[ Jul 24 21:42:01 Enabled. ] [ Jul 24 21:42:01 Rereading configuration. ] [ Jul 24 21:42:02 Rereading configuration. ] [ Jul 24 21:42:03 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:2::1]:5353 --dns-address [fd00:1122:3344:2::1]:53 &"). ] [ Jul 24 21:42:03 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:03.392371098Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"dns_config":"Config { bind_address: [fd00:1122:3344:2::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:03.392498151Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:03.415281254Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:03.415301695Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:03.415356606Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"component":"dns","local_address":"[fd00:1122:3344:2::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:03.415595763Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 55162,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:42:03.482476603Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::2]:56161","req_id":"28edc7f2-122c-4b12-b6b0-1702780e7756","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64338,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:42:03.75334432Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::2]:40055","req_id":"6387b591-b928-4eba-8905-88feec3441de","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 17205,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:42:04.452781356Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::2]:32320","req_id":"1e81c840-96e0-4972-b661-88d2bb0508a5","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:04.522748307Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:2::2]:57522"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:04.523432494Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"new_generation":1,"req_id":"3c309da2-12c2-455a-b116-18967609a9e9","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:04.523450455Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:04.539458141Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:04.539491332Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"new_generation":1,"req_id":"3c309da2-12c2-455a-b116-18967609a9e9","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:04.539498492Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"uri":"/config","method":"PUT","req_id":"3c309da2-12c2-455a-b116-18967609a9e9","remote_addr":"[fd00:1122:3344:2::2]:57522","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":16395,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 8056,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:42:15.416259211Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::1b]:47306","req_id":"99e14598-a3b4-4507-8fc9-b33e16d48bd3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 6166,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:42:15.420664516Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::1b]:64722","req_id":"e8678afb-88bb-4d6c-aee6-d4d25a1c0641","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 56568,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:42:15.424608218Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::1b]:58559","req_id":"1b278298-c935-4983-b9f2-b7f02087e5ce","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 9960,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:31.885536858Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::d]:65315","req_id":"7effaef0-89b8-4eb0-a6ef-b29992138410","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 60902,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:41.867156908Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::d]:53758","req_id":"8d77a05e-2ac3-4746-a691-54b4d576b9af","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 8037,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:42.178018216Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::c]:29407","req_id":"db0c7558-4ebf-4f2f-bd37-8ba862c49170","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 44261,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:43.431337263Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::b]:52167","req_id":"8a4401df-7883-4a27-93c7-79dc83361534","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52104,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:44.684712032Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::a]:33128","req_id":"f8d1e8f9-ee27-466d-ab57-db7761cf97bc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 10663,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 54,\n 52,\n 53,\n 99,\n 53,\n 102,\n 100,\n 102,\n 45,\n 53,\n 55,\n 56,\n 98,\n 45,\n 52,\n 97,\n 56,\n 54,\n 45,\n 56,\n 52,\n 57,\n 100,\n 45,\n 48,\n 57,\n 49,\n 55,\n 54,\n 50,\n 100,\n 102,\n 48,\n 55,\n 53,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:45.888169712Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::e]:43797","req_id":"66214ea2-ce5b-4dfc-bcb1-62c4e439e9d6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 10663,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 54,\n 52,\n 53,\n 99,\n 53,\n 102,\n 100,\n 102,\n 45,\n 53,\n 55,\n 56,\n 98,\n 45,\n 52,\n 97,\n 56,\n 54,\n 45,\n 56,\n 52,\n 57,\n 100,\n 45,\n 48,\n 57,\n 49,\n 55,\n 54,\n 50,\n 100,\n 102,\n 48,\n 55,\n 53,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:45.898755828Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::e]:51707","req_id":"31c0f7ca-192f-4645-8c7f-78cb1e970ed7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34068,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:51.868396629Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::d]:54798","req_id":"944fbfae-9c0e-41ea-ba6b-72ee5a83a2e5","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48828,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:01.871388297Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::d]:52501","req_id":"3618d362-b9e5-4153-a423-0703736e7a71","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 61364,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:11.873735869Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::d]:57283","req_id":"c3ebabbb-32e9-4460-9cee-505216261996","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4679,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:21.875423133Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::d]:57300","req_id":"ed2cd38c-bd9f-4d70-bd28-47f51a6976d0","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 55016,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:23.11489058Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::c]:20505","req_id":"5e681ad0-833b-477f-9045-adf09a3cbf5b","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:23.433737165Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:49962"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:23.495513613Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"uri":"/config","method":"GET","req_id":"2f72b861-73fd-4da0-916c-b6080746ba17","remote_addr":"[fd00:1122:3344:101::c]:49962","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":33019,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:26.632801615Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:55127"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:26.644298485Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"new_generation":1,"req_id":"abda35e0-d7b6-44cc-8e6b-d92fcaff08b9","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:26.644374236Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"new_generation":1,"req_id":"abda35e0-d7b6-44cc-8e6b-d92fcaff08b9","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:26.644381807Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"uri":"/config","method":"PUT","req_id":"abda35e0-d7b6-44cc-8e6b-d92fcaff08b9","remote_addr":"[fd00:1122:3344:101::c]:55127","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":10507,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42258,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:26.749661376Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::c]:11985","req_id":"79830011-f0d9-4a50-b159-8bdb1cffcb8e","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:26.863940589Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:65424"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:26.864184925Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"uri":"/config","method":"GET","req_id":"3b2c36fe-0e98-4316-9728-639cc254d94b","remote_addr":"[fd00:1122:3344:101::c]:65424","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":193,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16777,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:27.455353486Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::c]:64066","req_id":"7b72c056-c4be-4fd2-8ff7-052010b0550f","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:27.576729953Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:53858"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:27.57697682Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"uri":"/config","method":"GET","req_id":"e6d08652-e850-4b7a-9cba-1bb71f3de851","remote_addr":"[fd00:1122:3344:101::c]:53858","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":193,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16068,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:31.87724493Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::d]:54922","req_id":"bf1f5eea-1773-48e5-83d1-62be8178dbd1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 2143,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:34.150031211Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::b]:35532","req_id":"76b8338d-23e7-4256-8990-6d1819255c90","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:34.267879697Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:40582"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:34.268231627Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"uri":"/config","method":"GET","req_id":"38578b2d-22c5-4494-b45a-885e30b463c9","remote_addr":"[fd00:1122:3344:101::b]:40582","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":304,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 213,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:35.502800156Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::a]:57912","req_id":"dae94c6f-f19d-4bb7-b2ac-b41bd0375149","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:35.611205166Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:41419"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:35.611503604Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"uri":"/config","method":"GET","req_id":"31827f2b-f71f-4420-be5a-43c860a05655","remote_addr":"[fd00:1122:3344:101::a]:41419","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":257,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43296,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:41.879407176Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::d]:53991","req_id":"60030707-4caf-4b5f-bb92-eb45cee913d4","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:41.919773406Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:48828"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:41.921939952Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"new_generation":1,"req_id":"afe54c34-6870-4acb-ad61-0d750893699b","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:41.921988904Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"new_generation":1,"req_id":"afe54c34-6870-4acb-ad61-0d750893699b","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:41.921999534Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"uri":"/config","method":"PUT","req_id":"afe54c34-6870-4acb-ad61-0d750893699b","remote_addr":"[fd00:1122:3344:101::c]:48828","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":373,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41148,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:42.83660872Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::c]:28220","req_id":"8ff46c51-057b-4a13-b583-74c83b9633cb","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:42.936739005Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:40480"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:42.937090124Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"uri":"/config","method":"GET","req_id":"fded15ec-04aa-4c81-84d6-f4d22288460d","remote_addr":"[fd00:1122:3344:101::c]:40480","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":264,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:43.432909144Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:50201"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:43.434945717Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"new_generation":1,"req_id":"a7cfb47a-aac0-45c7-94f4-47447b111451","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:43.434997768Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"new_generation":1,"req_id":"a7cfb47a-aac0-45c7-94f4-47447b111451","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:43.435017409Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"uri":"/config","method":"PUT","req_id":"a7cfb47a-aac0-45c7-94f4-47447b111451","remote_addr":"[fd00:1122:3344:101::b]:50201","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":293,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48138,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:44.41940872Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::b]:46560","req_id":"abffe305-85b9-48ec-947d-e7230de2b7d2","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:44.517689046Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:44139"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:44.517922853Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"uri":"/config","method":"GET","req_id":"1e16ff30-0b92-459c-8b01-d4266f19b372","remote_addr":"[fd00:1122:3344:101::b]:44139","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":193,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:44.816651315Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:60365"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:44.818339899Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"new_generation":1,"req_id":"87155f64-ee9a-43f9-83af-449be012d7c3","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:44.818368269Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"new_generation":1,"req_id":"87155f64-ee9a-43f9-83af-449be012d7c3","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:44.81839211Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"uri":"/config","method":"PUT","req_id":"87155f64-ee9a-43f9-83af-449be012d7c3","remote_addr":"[fd00:1122:3344:101::a]:60365","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":265,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 56637,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:45.636174306Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::a]:44255","req_id":"c9f10502-1a94-4413-bbb8-8fe6442fea39","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:45.7331749Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:33710"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:45.733439317Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"uri":"/config","method":"GET","req_id":"90271842-47cb-4c74-bfcb-ac3036645406","remote_addr":"[fd00:1122:3344:101::a]:33710","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":201,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 8013,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:51.880672108Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::d]:52533","req_id":"c5d61785-43cb-41a9-9284-472f47dee353","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40945,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_e9266dde-d155-4ae7-915c-8af8a7eb59ad.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_e9266dde-d155-4ae7-915c-8af8a7eb59ad.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 101,\n 57,\n 50,\n 54,\n 54,\n 100,\n 100,\n 101,\n 45,\n 100,\n 49,\n 53,\n 53,\n 45,\n 52,\n 97,\n 101,\n 55,\n 45,\n 57,\n 49,\n 53,\n 99,\n 45,\n 56,\n 97,\n 102,\n 56,\n 97,\n 55,\n 101,\n 98,\n 53,\n 57,\n 97,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_e9266dde-d155-4ae7-915c-8af8a7eb59ad.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.028348226Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::4]:63738","req_id":"9ec87fad-c324-464a-be78-2c2c85f8f1fa","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40945,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_e9266dde-d155-4ae7-915c-8af8a7eb59ad.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_e9266dde-d155-4ae7-915c-8af8a7eb59ad.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 101,\n 57,\n 50,\n 54,\n 54,\n 100,\n 100,\n 101,\n 45,\n 100,\n 49,\n 53,\n 53,\n 45,\n 52,\n 97,\n 101,\n 55,\n 45,\n 57,\n 49,\n 53,\n 99,\n 45,\n 56,\n 97,\n 102,\n 56,\n 97,\n 55,\n 101,\n 98,\n 53,\n 57,\n 97,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_e9266dde-d155-4ae7-915c-8af8a7eb59ad.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.029169317Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::4]:62304","req_id":"18e77973-3179-4b0d-b749-91fca3cb1b8a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62078,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_4e16d07e-4a1c-4ea5-8a0a-2d13cf643967.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_4e16d07e-4a1c-4ea5-8a0a-2d13cf643967.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 52,\n 101,\n 49,\n 54,\n 100,\n 48,\n 55,\n 101,\n 45,\n 52,\n 97,\n 49,\n 99,\n 45,\n 52,\n 101,\n 97,\n 53,\n 45,\n 56,\n 97,\n 48,\n 97,\n 45,\n 50,\n 100,\n 49,\n 51,\n 99,\n 102,\n 54,\n 52,\n 51,\n 57,\n 54,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_4e16d07e-4a1c-4ea5-8a0a-2d13cf643967.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.069892857Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::5]:48765","req_id":"e29fc1cf-1f74-4f8f-91e3-41002f70ef3e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62078,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_4e16d07e-4a1c-4ea5-8a0a-2d13cf643967.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_4e16d07e-4a1c-4ea5-8a0a-2d13cf643967.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 52,\n 101,\n 49,\n 54,\n 100,\n 48,\n 55,\n 101,\n 45,\n 52,\n 97,\n 49,\n 99,\n 45,\n 52,\n 101,\n 97,\n 53,\n 45,\n 56,\n 97,\n 48,\n 97,\n 45,\n 50,\n 100,\n 49,\n 51,\n 99,\n 102,\n 54,\n 52,\n 51,\n 57,\n 54,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_4e16d07e-4a1c-4ea5-8a0a-2d13cf643967.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.070492722Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::5]:63173","req_id":"31789716-8f84-454f-891d-04700c62a2c8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52291,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 54,\n 52,\n 53,\n 99,\n 53,\n 102,\n 100,\n 102,\n 45,\n 53,\n 55,\n 56,\n 98,\n 45,\n 52,\n 97,\n 56,\n 54,\n 45,\n 56,\n 52,\n 57,\n 100,\n 45,\n 48,\n 57,\n 49,\n 55,\n 54,\n 50,\n 100,\n 102,\n 48,\n 55,\n 53,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.122477535Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::e]:41029","req_id":"b98b23ba-c00a-4567-b3fd-5a871c773adf","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52291,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 54,\n 52,\n 53,\n 99,\n 53,\n 102,\n 100,\n 102,\n 45,\n 53,\n 55,\n 56,\n 98,\n 45,\n 52,\n 97,\n 56,\n 54,\n 45,\n 56,\n 52,\n 57,\n 100,\n 45,\n 48,\n 57,\n 49,\n 55,\n 54,\n 50,\n 100,\n 102,\n 48,\n 55,\n 53,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.12304306Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::e]:34215","req_id":"657fa9ea-1a97-43fc-99ea-03e8b33e90b7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48816,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_ab61c6f0-8d7c-41ed-b2c9-5f8c105d1767.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_ab61c6f0-8d7c-41ed-b2c9-5f8c105d1767.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 97,\n 98,\n 54,\n 49,\n 99,\n 54,\n 102,\n 48,\n 45,\n 56,\n 100,\n 55,\n 99,\n 45,\n 52,\n 49,\n 101,\n 100,\n 45,\n 98,\n 50,\n 99,\n 57,\n 45,\n 53,\n 102,\n 56,\n 99,\n 49,\n 48,\n 53,\n 100,\n 49,\n 55,\n 54,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_ab61c6f0-8d7c-41ed-b2c9-5f8c105d1767.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.440968261Z","hostname":"oxz_internal_dns_116eb329-15f6-43cd-bd0c-cf7e6f2505a4","pid":5448,"peer_addr":"[fd00:1122:3344:101::1b]:54259","req_id":"04840eee-c6cf-49e5-925f-d7bdef30986b","component":"dns"}