[ May 15 02:41:28 Enabled. ] [ May 15 02:41:28 Rereading configuration. ] [ May 15 02:41:29 Rereading configuration. ] [ May 15 02:41:30 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:3::1]:5353 --dns-address [fd00:1122:3344:3::1]:53 &"). ] [ May 15 02:41:30 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:41:30.524612826Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"dns_config":"Config { bind_address: [fd00:1122:3344:3::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:41:30.524749269Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:41:30.549333888Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:41:30.549353638Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:41:30.54942065Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"component":"dns","local_address":"[fd00:1122:3344:3::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:41:30.549777347Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 439,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:41:30.586043833Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::2]:65514","req_id":"e8f2d74b-853e-4668-b95b-15b8977ded8c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18568,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:41:32.213073756Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:3::2]:48631","req_id":"cae36fec-7cab-4e21-b6b1-3487979152fd","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:41:33.181964696Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:3::2]:58367"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:41:33.182620629Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"new_generation":1,"req_id":"52115347-9d87-40f9-a4da-172db40a95e7","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:41:33.18265474Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:41:33.199602834Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:41:33.199633405Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"new_generation":1,"req_id":"52115347-9d87-40f9-a4da-172db40a95e7","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:41:33.199662405Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"uri":"/config","method":"PUT","req_id":"52115347-9d87-40f9-a4da-172db40a95e7","remote_addr":"[fd00:1122:3344:3::2]:58367","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":17385,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20980,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:41:44.807641633Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::1b]:34162","req_id":"8fc78402-acc9-4443-8909-9382be0b6a02","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30838,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:41:44.811999932Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::1b]:33157","req_id":"86ec2572-2147-4a6e-9701-34ef65c49b33","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 51980,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:41:44.816161116Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::1b]:48532","req_id":"39bc7f42-f5ee-4826-b0fa-1799dd73ab45","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38040,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:42:40.661554009Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::d]:49475","req_id":"3541f65a-106f-4972-9b62-3731cb748acc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46370,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_e2132fed-a6da-43e9-bcb0-17f7ac700304.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_e2132fed-a6da-43e9-bcb0-17f7ac700304.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 101,\n 50,\n 49,\n 51,\n 50,\n 102,\n 101,\n 100,\n 45,\n 97,\n 54,\n 100,\n 97,\n 45,\n 52,\n 51,\n 101,\n 57,\n 45,\n 98,\n 99,\n 98,\n 48,\n 45,\n 49,\n 55,\n 102,\n 55,\n 97,\n 99,\n 55,\n 48,\n 48,\n 51,\n 48,\n 52,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_e2132fed-a6da-43e9-bcb0-17f7ac700304.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:42:45.739036817Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::e]:58587","req_id":"f9621e1c-f673-4c98-8733-d079bc54ca56","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46370,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_e2132fed-a6da-43e9-bcb0-17f7ac700304.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_e2132fed-a6da-43e9-bcb0-17f7ac700304.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 101,\n 50,\n 49,\n 51,\n 50,\n 102,\n 101,\n 100,\n 45,\n 97,\n 54,\n 100,\n 97,\n 45,\n 52,\n 51,\n 101,\n 57,\n 45,\n 98,\n 99,\n 98,\n 48,\n 45,\n 49,\n 55,\n 102,\n 55,\n 97,\n 99,\n 55,\n 48,\n 48,\n 51,\n 48,\n 52,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_e2132fed-a6da-43e9-bcb0-17f7ac700304.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:42:45.739605998Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::e]:46117","req_id":"50639e5f-bf67-4988-a3c8-22d4afecdfb8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14963,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:42:50.636142661Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::d]:64710","req_id":"8b2fbfc6-f3fc-4fe8-950b-7bb1a3fbbdd9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23560,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:42:52.177973705Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::a]:2864","req_id":"b1a09b8a-86f3-46ad-b54b-3069566cf383","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34988,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:42:52.351967336Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::c]:47074","req_id":"64848a05-87c2-49c7-9cf2-703eecf07ae8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 65305,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:42:52.520521906Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::b]:62272","req_id":"2f94071f-dbf7-4d8f-85ca-e5baf921efd4","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:42:53.981358728Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:38512"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:42:54.048386938Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"new_generation":1,"req_id":"9b23e930-6152-46d4-8f99-3056da0e114a","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:42:54.048457539Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"new_generation":1,"req_id":"9b23e930-6152-46d4-8f99-3056da0e114a","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:42:54.04850743Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"uri":"/config","method":"PUT","req_id":"9b23e930-6152-46d4-8f99-3056da0e114a","remote_addr":"[fd00:1122:3344:101::b]:38512","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":27423,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 17504,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:42:54.154435209Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::b]:54151","req_id":"aa09059b-ca33-4029-b92b-6fb3964617b4","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:42:54.402061944Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:54078"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:42:54.411979205Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"uri":"/config","method":"GET","req_id":"23c1942e-85e6-4e7d-bffd-5f939ac09d05","remote_addr":"[fd00:1122:3344:101::b]:54078","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":9836,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 56685,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:42:55.014990342Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::b]:49219","req_id":"767efa75-c512-423f-a5de-05c5362e6014","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:42:55.149376348Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:41193"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:42:55.149630463Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"uri":"/config","method":"GET","req_id":"a3bcaaec-5b23-49c1-80d3-9997aaa6b9f2","remote_addr":"[fd00:1122:3344:101::b]:41193","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":196,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 55027,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:43:00.637525982Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::d]:56137","req_id":"62cceac5-b40e-41b2-9551-264a83e51c23","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42736,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:43:02.715518016Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::a]:14806","req_id":"5d372ea1-2673-419f-88aa-f5d7bf814ae1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 5329,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:43:02.839963441Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::c]:57009","req_id":"351f178e-62df-44fc-b77f-f13cab793ca4","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:02.93797127Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:65241"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:02.938262746Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"uri":"/config","method":"GET","req_id":"e5eb2165-3d00-4ad4-8386-c19dad6e0f40","remote_addr":"[fd00:1122:3344:101::a]:65241","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":227,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:02.992387124Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:46137"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:02.992599809Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"uri":"/config","method":"GET","req_id":"578a811a-a7ae-448d-a9e9-4d36385d7863","remote_addr":"[fd00:1122:3344:101::c]:46137","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":177,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 15000,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:43:10.638722055Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::d]:63467","req_id":"21d047f3-1f81-4ec7-90ff-e99bdfef1499","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18732,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:43:20.640748133Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::d]:61100","req_id":"34f8f904-0ba8-4e19-ac09-bd06f4f631f1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14991,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:43:20.678733884Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::1b]:36581","req_id":"0fa89d35-9e2c-4f83-af36-940a53eb1089","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 49974,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:43:20.683329307Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::1b]:47465","req_id":"03aa150a-282b-4b3b-a088-f23db737beb0","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23676,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:43:20.688144515Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::1b]:65378","req_id":"b2f7d231-b538-4c3d-b1c4-3f455a9ec124","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 35720,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:43:30.665165131Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::d]:63726","req_id":"9a989c5f-78c8-47cf-9d92-953ecfe3b036","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4500,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:43:40.667221085Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::d]:60621","req_id":"313832f7-b72f-4e59-95d2-e36c9ed8543c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14962,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:43:44.154755299Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::1b]:34708","req_id":"f6afdb90-0edd-46a2-8eea-1934138ae6c8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 22826,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:43:44.158908934Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::1b]:52198","req_id":"deac93b7-5ff0-47b6-9c94-98183011e5f4","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 37788,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:43:44.162798663Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::1b]:54248","req_id":"102e0e08-b62f-491e-a7e6-a287e6ec9f45","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 24061,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:43:50.668537956Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::d]:50844","req_id":"8c0346dc-1019-45f0-b50f-9d8e9a689756","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:52.168073673Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:41588"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:52.169933294Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"new_generation":1,"req_id":"1dc38be8-a6d1-430c-a341-0795c52827c4","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:52.169996795Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"new_generation":1,"req_id":"1dc38be8-a6d1-430c-a341-0795c52827c4","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:52.170013185Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"uri":"/config","method":"PUT","req_id":"1dc38be8-a6d1-430c-a341-0795c52827c4","remote_addr":"[fd00:1122:3344:101::a]:41588","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":205,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:52.20524277Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:55880"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:52.20735169Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"new_generation":1,"req_id":"95971f66-2621-4a95-96d8-917c4fa5e5b9","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:52.207373122Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"new_generation":1,"req_id":"95971f66-2621-4a95-96d8-917c4fa5e5b9","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:52.207391966Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"uri":"/config","method":"PUT","req_id":"95971f66-2621-4a95-96d8-917c4fa5e5b9","remote_addr":"[fd00:1122:3344:101::c]:55880","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":296,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:52.261482441Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:64489"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:52.263290208Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"new_generation":1,"req_id":"0f3486e2-8c53-4b98-97b3-034f89866ed1","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:52.263349649Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"new_generation":1,"req_id":"0f3486e2-8c53-4b98-97b3-034f89866ed1","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:52.263363249Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"uri":"/config","method":"PUT","req_id":"0f3486e2-8c53-4b98-97b3-034f89866ed1","remote_addr":"[fd00:1122:3344:101::b]:64489","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":378,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 25360,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:43:52.973791872Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::b]:4113","req_id":"defd9646-2cf5-462d-a99f-f0e3b3fab5b7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 9324,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:43:53.068955852Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::c]:64441","req_id":"26ec9625-c966-42d1-902c-a1dae9330d35","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:53.076525156Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:41096"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:53.076854613Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"uri":"/config","method":"GET","req_id":"d3971374-5d69-4206-b8f8-751c84307d51","remote_addr":"[fd00:1122:3344:101::b]:41096","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":267,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:53.27241257Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:43362"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:53.272667976Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"uri":"/config","method":"GET","req_id":"bab31f2c-8816-4638-8e4c-368b82c934b2","remote_addr":"[fd00:1122:3344:101::c]:43362","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":212,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43696,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:43:53.607833496Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::a]:1461","req_id":"9d35e19e-4b6f-4057-be5f-268ca494afb6","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:53.712252835Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:41340"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:43:53.712472989Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"uri":"/config","method":"GET","req_id":"c022e7a5-3d18-4a70-a410-e2bbf4e7f532","remote_addr":"[fd00:1122:3344:101::a]:41340","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":188,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 47145,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:44:00.669434458Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::d]:49345","req_id":"287a1e43-0be1-4f31-98fc-10f35d4b5f57","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 6718,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:44:10.670848283Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::d]:62842","req_id":"57468478-299d-4418-9d92-db1c52cdef2f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62963,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:44:20.673344368Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::d]:57925","req_id":"009915f2-e060-48b9-86cc-095c84fd4d9a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1069,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:44:30.674734546Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::d]:54888","req_id":"fac0319f-f68a-48f4-9de8-921eb988391b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16577,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:44:40.677190894Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::d]:49714","req_id":"87cc281d-f09a-4ea6-a53f-bc3c5eecc3d8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 19387,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:44:50.678515953Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::d]:58820","req_id":"407adbee-343f-4a67-925c-aa29de78b480","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:44:51.953995194Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:57966"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:44:51.955338731Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"new_generation":1,"req_id":"f5c2ad43-25b1-498b-a91c-1b89a93109ba","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:44:51.955366102Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"new_generation":1,"req_id":"f5c2ad43-25b1-498b-a91c-1b89a93109ba","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:44:51.955472004Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"uri":"/config","method":"PUT","req_id":"f5c2ad43-25b1-498b-a91c-1b89a93109ba","remote_addr":"[fd00:1122:3344:101::a]:57966","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":196,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:44:52.054549564Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:58598"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:44:52.056272982Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"new_generation":1,"req_id":"a0b19833-3112-4d7f-8d69-0697dcca286a","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:44:52.056409082Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"new_generation":1,"req_id":"a0b19833-3112-4d7f-8d69-0697dcca286a","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:44:52.056421888Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"uri":"/config","method":"PUT","req_id":"a0b19833-3112-4d7f-8d69-0697dcca286a","remote_addr":"[fd00:1122:3344:101::c]:58598","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":237,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:44:52.262443912Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:36469"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:44:52.264529989Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"new_generation":1,"req_id":"799dbc7c-422f-4b2c-ae2f-1571288fd62b","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:44:52.264570761Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"new_generation":1,"req_id":"799dbc7c-422f-4b2c-ae2f-1571288fd62b","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:44:52.264696502Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"uri":"/config","method":"PUT","req_id":"799dbc7c-422f-4b2c-ae2f-1571288fd62b","remote_addr":"[fd00:1122:3344:101::b]:36469","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":285,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 27561,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:44:53.050323216Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::a]:25952","req_id":"373869f2-27ae-4168-84c2-751bcf2a02bb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 12677,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:44:53.074007179Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::b]:3201","req_id":"375b4a74-a41f-42f2-bce6-5da60134c6c3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50506,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:44:53.077573841Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::c]:42960","req_id":"7bf2ab0b-7518-40c6-9566-86956829ab3b","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:44:53.16529599Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:37689"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:44:53.165526905Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"uri":"/config","method":"GET","req_id":"a6554fb1-9194-4f4f-86a9-9ae924f6e696","remote_addr":"[fd00:1122:3344:101::a]:37689","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":186,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:44:53.203295111Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:39723"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:44:53.203522716Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"uri":"/config","method":"GET","req_id":"2dd70a29-f2b0-4399-bf28-3d609da054cc","remote_addr":"[fd00:1122:3344:101::b]:39723","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":192,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:44:53.243499647Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:36269"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-15T02:44:53.243787803Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"uri":"/config","method":"GET","req_id":"90aeeb30-3ee2-4c6c-b7a8-723de893f03b","remote_addr":"[fd00:1122:3344:101::c]:36269","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":244,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 919,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_91005d09-cc59-416f-aee6-aa65462b0fb0.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_91005d09-cc59-416f-aee6-aa65462b0fb0.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 57,\n 49,\n 48,\n 48,\n 53,\n 100,\n 48,\n 57,\n 45,\n 99,\n 99,\n 53,\n 57,\n 45,\n 52,\n 49,\n 54,\n 102,\n 45,\n 97,\n 101,\n 101,\n 54,\n 45,\n 97,\n 97,\n 54,\n 53,\n 52,\n 54,\n 50,\n 98,\n 48,\n 102,\n 98,\n 48,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_91005d09-cc59-416f-aee6-aa65462b0fb0.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:45:00.08060543Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::7]:37181","req_id":"7c1fd5a9-7acb-4376-897a-3b5003f10ea8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 919,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_91005d09-cc59-416f-aee6-aa65462b0fb0.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_91005d09-cc59-416f-aee6-aa65462b0fb0.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 57,\n 49,\n 48,\n 48,\n 53,\n 100,\n 48,\n 57,\n 45,\n 99,\n 99,\n 53,\n 57,\n 45,\n 52,\n 49,\n 54,\n 102,\n 45,\n 97,\n 101,\n 101,\n 54,\n 45,\n 97,\n 97,\n 54,\n 53,\n 52,\n 54,\n 50,\n 98,\n 48,\n 102,\n 98,\n 48,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_91005d09-cc59-416f-aee6-aa65462b0fb0.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:45:00.081222982Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::7]:39588","req_id":"e4488677-e524-4a13-8d8d-5bad998c5ede","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48635,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:45:00.312978804Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::2]:59572","req_id":"98e471c7-cb00-41b1-8fae-46427fef5a5d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48635,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:45:00.313517205Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::2]:38609","req_id":"548fd62b-2c69-4347-859b-f5a06ce5dc71","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 63464,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_e193044e-aa73-41de-b4ce-b08357a3d432.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_e193044e-aa73-41de-b4ce-b08357a3d432.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 101,\n 49,\n 57,\n 51,\n 48,\n 52,\n 52,\n 101,\n 45,\n 97,\n 97,\n 55,\n 51,\n 45,\n 52,\n 49,\n 100,\n 101,\n 45,\n 98,\n 52,\n 99,\n 101,\n 45,\n 98,\n 48,\n 56,\n 51,\n 53,\n 55,\n 97,\n 51,\n 100,\n 52,\n 51,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_e193044e-aa73-41de-b4ce-b08357a3d432.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:45:00.442988842Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::6]:41561","req_id":"d3468b7c-66ba-4c56-ba6a-a2f6df51553c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 63464,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_e193044e-aa73-41de-b4ce-b08357a3d432.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_e193044e-aa73-41de-b4ce-b08357a3d432.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 101,\n 49,\n 57,\n 51,\n 48,\n 52,\n 52,\n 101,\n 45,\n 97,\n 97,\n 55,\n 51,\n 45,\n 52,\n 49,\n 100,\n 101,\n 45,\n 98,\n 52,\n 99,\n 101,\n 45,\n 98,\n 48,\n 56,\n 51,\n 53,\n 55,\n 97,\n 51,\n 100,\n 52,\n 51,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_e193044e-aa73-41de-b4ce-b08357a3d432.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:45:00.443620975Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::6]:46879","req_id":"756057c7-82a6-40f1-a9ed-2105f4b9c084","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53597,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_3bf717fa-c147-4091-a0bc-b678be30cccd.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_3bf717fa-c147-4091-a0bc-b678be30cccd.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 51,\n 98,\n 102,\n 55,\n 49,\n 55,\n 102,\n 97,\n 45,\n 99,\n 49,\n 52,\n 55,\n 45,\n 52,\n 48,\n 57,\n 49,\n 45,\n 97,\n 48,\n 98,\n 99,\n 45,\n 98,\n 54,\n 55,\n 56,\n 98,\n 101,\n 51,\n 48,\n 99,\n 99,\n 99,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_3bf717fa-c147-4091-a0bc-b678be30cccd.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:45:00.48919978Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::1b]:36898","req_id":"9b578f79-9604-45fe-8820-aeb2d7ffd3c0","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57924,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_d92cf641-3430-4ac3-92b3-32d39db862be.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_d92cf641-3430-4ac3-92b3-32d39db862be.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 100,\n 57,\n 50,\n 99,\n 102,\n 54,\n 52,\n 49,\n 45,\n 51,\n 52,\n 51,\n 48,\n 45,\n 52,\n 97,\n 99,\n 51,\n 45,\n 57,\n 50,\n 98,\n 51,\n 45,\n 51,\n 50,\n 100,\n 51,\n 57,\n 100,\n 98,\n 56,\n 54,\n 50,\n 98,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_d92cf641-3430-4ac3-92b3-32d39db862be.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:45:00.569788025Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::4]:44621","req_id":"dacbce40-6a6c-4962-a521-e1bc4a2ed11f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57924,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_d92cf641-3430-4ac3-92b3-32d39db862be.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_d92cf641-3430-4ac3-92b3-32d39db862be.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 100,\n 57,\n 50,\n 99,\n 102,\n 54,\n 52,\n 49,\n 45,\n 51,\n 52,\n 51,\n 48,\n 45,\n 52,\n 97,\n 99,\n 51,\n 45,\n 57,\n 50,\n 98,\n 51,\n 45,\n 51,\n 50,\n 100,\n 51,\n 57,\n 100,\n 98,\n 56,\n 54,\n 50,\n 98,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_d92cf641-3430-4ac3-92b3-32d39db862be.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:45:00.57051696Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::4]:53367","req_id":"940c9854-c3e0-47f2-a161-392cc1a61292","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 26823,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_b22828b8-2881-4adf-80d6-577e84d783a3.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_b22828b8-2881-4adf-80d6-577e84d783a3.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 50,\n 50,\n 56,\n 50,\n 56,\n 98,\n 56,\n 45,\n 50,\n 56,\n 56,\n 49,\n 45,\n 52,\n 97,\n 100,\n 102,\n 45,\n 56,\n 48,\n 100,\n 54,\n 45,\n 53,\n 55,\n 55,\n 101,\n 56,\n 52,\n 100,\n 55,\n 56,\n 51,\n 97,\n 51,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_b22828b8-2881-4adf-80d6-577e84d783a3.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:45:00.582556814Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::3]:60990","req_id":"e2433188-f142-44c1-9e14-5fbafdcc838d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 26823,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_b22828b8-2881-4adf-80d6-577e84d783a3.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_b22828b8-2881-4adf-80d6-577e84d783a3.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 50,\n 50,\n 56,\n 50,\n 56,\n 98,\n 56,\n 45,\n 50,\n 56,\n 56,\n 49,\n 45,\n 52,\n 97,\n 100,\n 102,\n 45,\n 56,\n 48,\n 100,\n 54,\n 45,\n 53,\n 55,\n 55,\n 101,\n 56,\n 52,\n 100,\n 55,\n 56,\n 51,\n 97,\n 51,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_b22828b8-2881-4adf-80d6-577e84d783a3.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:45:00.583209248Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::3]:35034","req_id":"a3396504-3f98-45a3-b95a-389c66dd909a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 32830,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_65eec142-5807-438b-b65a-2b349a4d4cfa.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_65eec142-5807-438b-b65a-2b349a4d4cfa.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 53,\n 101,\n 101,\n 99,\n 49,\n 52,\n 50,\n 45,\n 53,\n 56,\n 48,\n 55,\n 45,\n 52,\n 51,\n 56,\n 98,\n 45,\n 98,\n 54,\n 53,\n 97,\n 45,\n 50,\n 98,\n 51,\n 52,\n 57,\n 97,\n 52,\n 100,\n 52,\n 99,\n 102,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_65eec142-5807-438b-b65a-2b349a4d4cfa.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:45:00.588917554Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::5]:36981","req_id":"f7d0082d-5e2f-48d4-9864-c814bcf79af8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 32830,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_65eec142-5807-438b-b65a-2b349a4d4cfa.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_65eec142-5807-438b-b65a-2b349a4d4cfa.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 53,\n 101,\n 101,\n 99,\n 49,\n 52,\n 50,\n 45,\n 53,\n 56,\n 48,\n 55,\n 45,\n 52,\n 51,\n 56,\n 98,\n 45,\n 98,\n 54,\n 53,\n 97,\n 45,\n 50,\n 98,\n 51,\n 52,\n 57,\n 97,\n 52,\n 100,\n 52,\n 99,\n 102,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_65eec142-5807-438b-b65a-2b349a4d4cfa.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:45:00.589421334Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::5]:35607","req_id":"36f9bd72-3dc0-4517-bcb1-d05f4f0a2176","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41989,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:45:00.680512422Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::d]:64740","req_id":"5ff02a23-7560-49a8-a82b-9a87fa356efc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 29112,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_e2132fed-a6da-43e9-bcb0-17f7ac700304.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_e2132fed-a6da-43e9-bcb0-17f7ac700304.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 101,\n 50,\n 49,\n 51,\n 50,\n 102,\n 101,\n 100,\n 45,\n 97,\n 54,\n 100,\n 97,\n 45,\n 52,\n 51,\n 101,\n 57,\n 45,\n 98,\n 99,\n 98,\n 48,\n 45,\n 49,\n 55,\n 102,\n 55,\n 97,\n 99,\n 55,\n 48,\n 48,\n 51,\n 48,\n 52,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_e2132fed-a6da-43e9-bcb0-17f7ac700304.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:45:00.681987542Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::e]:54320","req_id":"779ca0b1-a4a8-4503-a860-b7338b9498fe","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 29112,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_e2132fed-a6da-43e9-bcb0-17f7ac700304.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_e2132fed-a6da-43e9-bcb0-17f7ac700304.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 101,\n 50,\n 49,\n 51,\n 50,\n 102,\n 101,\n 100,\n 45,\n 97,\n 54,\n 100,\n 97,\n 45,\n 52,\n 51,\n 101,\n 57,\n 45,\n 98,\n 99,\n 98,\n 48,\n 45,\n 49,\n 55,\n 102,\n 55,\n 97,\n 99,\n 55,\n 48,\n 48,\n 51,\n 48,\n 52,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_e2132fed-a6da-43e9-bcb0-17f7ac700304.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-15T02:45:00.682666056Z","hostname":"oxz_internal_dns_b739b0b8-627f-4eed-903a-f5f83e038d9c","pid":5424,"peer_addr":"[fd00:1122:3344:101::e]:37059","req_id":"06b6146e-a237-4980-8c93-143c50c3a379","component":"dns"}