[ Jul 24 21:42:01 Enabled. ] [ Jul 24 21:42:01 Rereading configuration. ] [ Jul 24 21:42:02 Rereading configuration. ] [ Jul 24 21:42:03 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:1::1]:5353 --dns-address [fd00:1122:3344:1::1]:53 &"). ] [ Jul 24 21:42:03 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:1::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:03.418373295Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"dns_config":"Config { bind_address: [fd00:1122:3344:1::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:1::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:03.418493248Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:03.443413106Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:03.443431027Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:03.443490048Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"component":"dns","local_address":"[fd00:1122:3344:1::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:03.443797286Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 49820,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:42:03.482505773Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::2]:35754","req_id":"7236f410-4837-46cc-b6fc-85f45281da5c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31646,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:42:03.753311109Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::2]:49464","req_id":"b9ab4031-4fd7-49be-a4f8-0b868e14db46","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 15965,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:42:04.453150056Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::2]:13741","req_id":"26445ccf-c669-4913-a7bf-3b05471abd7f","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:04.540780846Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:1::2]:44340"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:04.541463273Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"new_generation":1,"req_id":"622ce614-784b-4e9d-801b-83a01c36e65f","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:04.541490664Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:04.55748227Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:04.557516401Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"new_generation":1,"req_id":"622ce614-784b-4e9d-801b-83a01c36e65f","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:04.557523611Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"uri":"/config","method":"PUT","req_id":"622ce614-784b-4e9d-801b-83a01c36e65f","remote_addr":"[fd00:1122:3344:1::2]:44340","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":16418,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 8056,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:42:15.415910292Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::1b]:44826","req_id":"061b0b57-60d6-4440-b8b4-ca462272b8da","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 6166,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:42:15.420475021Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::1b]:51236","req_id":"aeb4be8c-50d9-4fb7-92dc-04c183e49d85","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 56568,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:42:15.424509586Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::1b]:58933","req_id":"a6769a55-a756-41c6-8139-7acdebfc90d5","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42189,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:31.885139247Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::d]:49982","req_id":"b8ecc95b-d8de-4b20-b3cf-f9ffba989681","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62524,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:41.867158939Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::d]:60653","req_id":"985f2e28-6747-4013-8923-434309b8bca1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48108,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:42.178121819Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::c]:11138","req_id":"e76fa89e-b330-405d-80f4-055706163b9e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 27799,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:43.430881591Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::b]:44765","req_id":"a6cd3b92-dcbe-4870-978c-a5783d95aa4a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21223,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:44.683215753Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::a]:4864","req_id":"3382444d-fa37-46f1-8040-307891423fc7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 10663,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 54,\n 52,\n 53,\n 99,\n 53,\n 102,\n 100,\n 102,\n 45,\n 53,\n 55,\n 56,\n 98,\n 45,\n 52,\n 97,\n 56,\n 54,\n 45,\n 56,\n 52,\n 57,\n 100,\n 45,\n 48,\n 57,\n 49,\n 55,\n 54,\n 50,\n 100,\n 102,\n 48,\n 55,\n 53,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:45.878327706Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::e]:43326","req_id":"063c9e4a-c9a2-4dd7-99a8-37715c0605d8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 10663,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 54,\n 52,\n 53,\n 99,\n 53,\n 102,\n 100,\n 102,\n 45,\n 53,\n 55,\n 56,\n 98,\n 45,\n 52,\n 97,\n 56,\n 54,\n 45,\n 56,\n 52,\n 57,\n 100,\n 45,\n 48,\n 57,\n 49,\n 55,\n 54,\n 50,\n 100,\n 102,\n 48,\n 55,\n 53,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:45.898444249Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::e]:61996","req_id":"887ae74b-d648-4c31-8681-46085372b06e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 56495,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:51.868375839Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::d]:53552","req_id":"22ef1b76-9d1d-4985-a32f-70381df9e02b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21438,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:01.871043008Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::d]:61360","req_id":"e2159d5a-5160-4d2c-a00b-5d6aa8f8166d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59426,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:11.873732139Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::d]:55560","req_id":"994db802-c44f-4d9f-b6a4-a22234e8840f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 9655,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:21.875440223Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::d]:50837","req_id":"7f51ccb5-eb8f-4546-96fa-2ca350aa3a56","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59522,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:23.11488945Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::c]:63290","req_id":"9fd966bd-1e9d-4f0b-a8f1-574f5943ac12","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:23.507806563Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:36572"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:23.559649031Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"uri":"/config","method":"GET","req_id":"97cbcfb7-b151-4e70-a842-dbc7efad5aa2","remote_addr":"[fd00:1122:3344:101::c]:36572","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":31580,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:26.6337608Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:39703"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:26.64183048Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"new_generation":1,"req_id":"6fda4d40-f3ec-4848-9f4a-dae2d1880bc0","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:26.641924483Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"new_generation":1,"req_id":"6fda4d40-f3ec-4848-9f4a-dae2d1880bc0","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:26.641933663Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"uri":"/config","method":"PUT","req_id":"6fda4d40-f3ec-4848-9f4a-dae2d1880bc0","remote_addr":"[fd00:1122:3344:101::c]:39703","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":7920,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 49277,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:26.749732277Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::c]:35841","req_id":"6d475465-bb11-449b-9e52-2737fcb6dc9d","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:26.864605676Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:57363"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:26.864913684Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"uri":"/config","method":"GET","req_id":"eef6ded6-e00d-4cc2-a91f-c088d06697a6","remote_addr":"[fd00:1122:3344:101::c]:57363","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":234,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3125,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:27.455342295Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::c]:4484","req_id":"cd76123d-7df8-4c0c-b94c-54c9de49b9c4","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:27.57738527Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:63118"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:27.577644917Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"uri":"/config","method":"GET","req_id":"36f7d87d-a826-4baf-842e-34a5b831346d","remote_addr":"[fd00:1122:3344:101::c]:63118","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":201,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43151,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:31.877276321Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::d]:60923","req_id":"25bdda76-836e-4a9c-bd75-2ffd6a38c369","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43471,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:34.150016631Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::b]:42385","req_id":"8d88f983-52c2-4e00-868a-4120cac968a7","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:34.268696059Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:33064"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:34.268953885Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"uri":"/config","method":"GET","req_id":"ddda74bc-da77-4e97-a0b6-5bd2c6d7f972","remote_addr":"[fd00:1122:3344:101::b]:33064","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":188,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 13170,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:35.502875018Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::a]:34378","req_id":"c1e922a2-507e-4384-87c7-6c317041bad8","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:35.612032608Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:36738"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:35.612284765Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"uri":"/config","method":"GET","req_id":"992bd5a7-3dd9-4237-98b4-4bab9f524985","remote_addr":"[fd00:1122:3344:101::a]:36738","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":197,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 11746,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:41.879437267Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::d]:50890","req_id":"d0eaeb04-4398-4889-a49c-fa4d06b0a8ad","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:41.921259945Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:57272"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:41.92184468Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"new_generation":1,"req_id":"ccdde419-941e-40fc-8519-33b5b36a1d1e","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:41.921898391Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"new_generation":1,"req_id":"ccdde419-941e-40fc-8519-33b5b36a1d1e","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:41.921933542Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"uri":"/config","method":"PUT","req_id":"ccdde419-941e-40fc-8519-33b5b36a1d1e","remote_addr":"[fd00:1122:3344:101::c]:57272","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":257,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 27178,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:42.836658531Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::c]:23934","req_id":"471c5dff-20b4-4201-b766-af1929523a36","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:42.937525345Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:63987"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:42.937736991Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"uri":"/config","method":"GET","req_id":"2ce93748-ac54-4c3f-aa23-37deecdeed59","remote_addr":"[fd00:1122:3344:101::c]:63987","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":176,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:43.434586317Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:62613"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:43.434929706Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"new_generation":1,"req_id":"f2709032-0771-47d8-9f11-94771763f2bb","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:43.434972597Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"new_generation":1,"req_id":"f2709032-0771-47d8-9f11-94771763f2bb","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:43.434992948Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"uri":"/config","method":"PUT","req_id":"f2709032-0771-47d8-9f11-94771763f2bb","remote_addr":"[fd00:1122:3344:101::b]:62613","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":207,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46697,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:44.419392569Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::b]:53374","req_id":"dc207748-6fa1-487a-84c0-9aa449a29261","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:44.518352904Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:39043"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:44.5185714Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"uri":"/config","method":"GET","req_id":"3bf3da1f-b3b1-4f7f-a43c-dc52765aefa1","remote_addr":"[fd00:1122:3344:101::b]:39043","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":183,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:44.818048041Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:44772"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:44.818415241Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"new_generation":1,"req_id":"091e9f8d-b2b1-4086-b9a2-5d4ca15418ce","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:44.818432001Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"new_generation":1,"req_id":"091e9f8d-b2b1-4086-b9a2-5d4ca15418ce","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:44.818451951Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"uri":"/config","method":"PUT","req_id":"091e9f8d-b2b1-4086-b9a2-5d4ca15418ce","remote_addr":"[fd00:1122:3344:101::a]:44772","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":290,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20651,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:45.636176876Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::a]:55513","req_id":"4d37efd3-712b-48a4-b87f-f5d34483b658","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:45.733869938Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:63206"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:45.734108144Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"uri":"/config","method":"GET","req_id":"a2dd9100-2201-4d69-95f0-a12382b382dc","remote_addr":"[fd00:1122:3344:101::a]:63206","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":188,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38394,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:51.880682508Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::d]:59742","req_id":"e72626eb-6490-4190-8861-b2739d660feb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40945,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_e9266dde-d155-4ae7-915c-8af8a7eb59ad.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_e9266dde-d155-4ae7-915c-8af8a7eb59ad.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 101,\n 57,\n 50,\n 54,\n 54,\n 100,\n 100,\n 101,\n 45,\n 100,\n 49,\n 53,\n 53,\n 45,\n 52,\n 97,\n 101,\n 55,\n 45,\n 57,\n 49,\n 53,\n 99,\n 45,\n 56,\n 97,\n 102,\n 56,\n 97,\n 55,\n 101,\n 98,\n 53,\n 57,\n 97,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_e9266dde-d155-4ae7-915c-8af8a7eb59ad.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.027993456Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::4]:61354","req_id":"2ae67d75-178d-4cd4-ab97-ae7fc6844d9a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40945,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_e9266dde-d155-4ae7-915c-8af8a7eb59ad.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_e9266dde-d155-4ae7-915c-8af8a7eb59ad.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 101,\n 57,\n 50,\n 54,\n 54,\n 100,\n 100,\n 101,\n 45,\n 100,\n 49,\n 53,\n 53,\n 45,\n 52,\n 97,\n 101,\n 55,\n 45,\n 57,\n 49,\n 53,\n 99,\n 45,\n 56,\n 97,\n 102,\n 56,\n 97,\n 55,\n 101,\n 98,\n 53,\n 57,\n 97,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_e9266dde-d155-4ae7-915c-8af8a7eb59ad.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.028923651Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::4]:37328","req_id":"824b209e-7d1c-4a74-9883-1683715a3a4d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62078,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_4e16d07e-4a1c-4ea5-8a0a-2d13cf643967.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_4e16d07e-4a1c-4ea5-8a0a-2d13cf643967.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 52,\n 101,\n 49,\n 54,\n 100,\n 48,\n 55,\n 101,\n 45,\n 52,\n 97,\n 49,\n 99,\n 45,\n 52,\n 101,\n 97,\n 53,\n 45,\n 56,\n 97,\n 48,\n 97,\n 45,\n 50,\n 100,\n 49,\n 51,\n 99,\n 102,\n 54,\n 52,\n 51,\n 57,\n 54,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_4e16d07e-4a1c-4ea5-8a0a-2d13cf643967.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.069620459Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::5]:33198","req_id":"978e5e92-66b1-4168-a107-1f0030f64433","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62078,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_4e16d07e-4a1c-4ea5-8a0a-2d13cf643967.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_4e16d07e-4a1c-4ea5-8a0a-2d13cf643967.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 52,\n 101,\n 49,\n 54,\n 100,\n 48,\n 55,\n 101,\n 45,\n 52,\n 97,\n 49,\n 99,\n 45,\n 52,\n 101,\n 97,\n 53,\n 45,\n 56,\n 97,\n 48,\n 97,\n 45,\n 50,\n 100,\n 49,\n 51,\n 99,\n 102,\n 54,\n 52,\n 51,\n 57,\n 54,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_4e16d07e-4a1c-4ea5-8a0a-2d13cf643967.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.070292817Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::5]:36595","req_id":"a5e5aeb7-d832-458f-817e-5513ea0f2b06","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52291,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 54,\n 52,\n 53,\n 99,\n 53,\n 102,\n 100,\n 102,\n 45,\n 53,\n 55,\n 56,\n 98,\n 45,\n 52,\n 97,\n 56,\n 54,\n 45,\n 56,\n 52,\n 57,\n 100,\n 45,\n 48,\n 57,\n 49,\n 55,\n 54,\n 50,\n 100,\n 102,\n 48,\n 55,\n 53,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.1222854Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::e]:50725","req_id":"3d46d7cf-4766-4436-89fc-7f356594c19c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52291,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 54,\n 52,\n 53,\n 99,\n 53,\n 102,\n 100,\n 102,\n 45,\n 53,\n 55,\n 56,\n 98,\n 45,\n 52,\n 97,\n 56,\n 54,\n 45,\n 56,\n 52,\n 57,\n 100,\n 45,\n 48,\n 57,\n 49,\n 55,\n 54,\n 50,\n 100,\n 102,\n 48,\n 55,\n 53,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.122819784Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::e]:43045","req_id":"ba138d68-b8ba-4a58-9efd-75dd97b3d630","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48816,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_ab61c6f0-8d7c-41ed-b2c9-5f8c105d1767.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_ab61c6f0-8d7c-41ed-b2c9-5f8c105d1767.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 97,\n 98,\n 54,\n 49,\n 99,\n 54,\n 102,\n 48,\n 45,\n 56,\n 100,\n 55,\n 99,\n 45,\n 52,\n 49,\n 101,\n 100,\n 45,\n 98,\n 50,\n 99,\n 57,\n 45,\n 53,\n 102,\n 56,\n 99,\n 49,\n 48,\n 53,\n 100,\n 49,\n 55,\n 54,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_ab61c6f0-8d7c-41ed-b2c9-5f8c105d1767.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.440715024Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::1b]:35986","req_id":"0fa780f4-31a0-4f94-9002-2977b9063cb7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 47032,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_b18bd539-4d7c-41b4-93ac-99b66585e432.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_b18bd539-4d7c-41b4-93ac-99b66585e432.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 49,\n 56,\n 98,\n 100,\n 53,\n 51,\n 57,\n 45,\n 52,\n 100,\n 55,\n 99,\n 45,\n 52,\n 49,\n 98,\n 52,\n 45,\n 57,\n 51,\n 97,\n 99,\n 45,\n 57,\n 57,\n 98,\n 54,\n 54,\n 53,\n 56,\n 53,\n 101,\n 52,\n 51,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_b18bd539-4d7c-41b4-93ac-99b66585e432.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.780408072Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::6]:35450","req_id":"01e48da4-d5c9-473f-a6ed-a0f754de85e4","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 47032,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_b18bd539-4d7c-41b4-93ac-99b66585e432.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_b18bd539-4d7c-41b4-93ac-99b66585e432.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 49,\n 56,\n 98,\n 100,\n 53,\n 51,\n 57,\n 45,\n 52,\n 100,\n 55,\n 99,\n 45,\n 52,\n 49,\n 98,\n 52,\n 45,\n 57,\n 51,\n 97,\n 99,\n 45,\n 57,\n 57,\n 98,\n 54,\n 54,\n 53,\n 56,\n 53,\n 101,\n 52,\n 51,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_b18bd539-4d7c-41b4-93ac-99b66585e432.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.781295535Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::6]:35250","req_id":"e971a41d-e0c2-48b4-80d2-9187aae58805","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57674,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_c53bb4cd-f9ba-4262-a310-9aab9e790d29.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_c53bb4cd-f9ba-4262-a310-9aab9e790d29.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 99,\n 53,\n 51,\n 98,\n 98,\n 52,\n 99,\n 100,\n 45,\n 102,\n 57,\n 98,\n 97,\n 45,\n 52,\n 50,\n 54,\n 50,\n 45,\n 97,\n 51,\n 49,\n 48,\n 45,\n 57,\n 97,\n 97,\n 98,\n 57,\n 101,\n 55,\n 57,\n 48,\n 100,\n 50,\n 57,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_c53bb4cd-f9ba-4262-a310-9aab9e790d29.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.800997628Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::7]:40151","req_id":"8e4a908d-cd7e-4883-a26f-9fd77acbbcff","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57674,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_c53bb4cd-f9ba-4262-a310-9aab9e790d29.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_c53bb4cd-f9ba-4262-a310-9aab9e790d29.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 99,\n 53,\n 51,\n 98,\n 98,\n 52,\n 99,\n 100,\n 45,\n 102,\n 57,\n 98,\n 97,\n 45,\n 52,\n 50,\n 54,\n 50,\n 45,\n 97,\n 51,\n 49,\n 48,\n 45,\n 57,\n 97,\n 97,\n 98,\n 57,\n 101,\n 55,\n 57,\n 48,\n 100,\n 50,\n 57,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_c53bb4cd-f9ba-4262-a310-9aab9e790d29.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.801739397Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::7]:62302","req_id":"10497f48-805b-4ed9-a443-0ced7d45de47","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50659,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.891170263Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::2]:35774","req_id":"5c126b8b-14c6-4a6d-a679-d6e35bf1239c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50659,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.89180464Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::2]:43479","req_id":"6a7a7ce1-7aec-4457-9936-4c412a2e7e18","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64365,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_81c0d863-a7d9-422c-93e2-a5815214b3d5.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_81c0d863-a7d9-422c-93e2-a5815214b3d5.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 49,\n 99,\n 48,\n 100,\n 56,\n 54,\n 51,\n 45,\n 97,\n 55,\n 100,\n 57,\n 45,\n 52,\n 50,\n 50,\n 99,\n 45,\n 57,\n 51,\n 101,\n 50,\n 45,\n 97,\n 53,\n 56,\n 49,\n 53,\n 50,\n 49,\n 52,\n 98,\n 51,\n 100,\n 53,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_81c0d863-a7d9-422c-93e2-a5815214b3d5.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:01.009986044Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::3]:42247","req_id":"9632155a-a7fa-4b3a-ab79-781bbaf489e6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64365,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_81c0d863-a7d9-422c-93e2-a5815214b3d5.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_81c0d863-a7d9-422c-93e2-a5815214b3d5.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 49,\n 99,\n 48,\n 100,\n 56,\n 54,\n 51,\n 45,\n 97,\n 55,\n 100,\n 57,\n 45,\n 52,\n 50,\n 50,\n 99,\n 45,\n 57,\n 51,\n 101,\n 50,\n 45,\n 97,\n 53,\n 56,\n 49,\n 53,\n 50,\n 49,\n 52,\n 98,\n 51,\n 100,\n 53,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_81c0d863-a7d9-422c-93e2-a5815214b3d5.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:01.010723254Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::3]:54735","req_id":"918c4eb1-aacf-483d-b55c-80f38e211533","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 55663,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:01.881742465Z","hostname":"oxz_internal_dns_90d1f2b5-f8cc-4219-8c0d-ea2e2e580a1d","pid":5454,"peer_addr":"[fd00:1122:3344:101::d]:55369","req_id":"cf7a08e0-9d96-4c6b-a3de-c1a26cc053cd","component":"dns"}