[ Jul 24 21:42:01 Enabled. ] [ Jul 24 21:42:01 Rereading configuration. ] [ Jul 24 21:42:02 Rereading configuration. ] [ Jul 24 21:42:03 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:3::1]:5353 --dns-address [fd00:1122:3344:3::1]:53 &"). ] [ Jul 24 21:42:03 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:03.422446421Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"dns_config":"Config { bind_address: [fd00:1122:3344:3::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:03.422566094Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:03.445843399Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:03.44585546Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:03.445916751Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"component":"dns","local_address":"[fd00:1122:3344:3::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:03.446159748Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50528,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:42:03.482675178Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::2]:5642","req_id":"b5717fb6-d405-42a4-94e9-5a6aaf24fc6f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3641,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:42:03.753297458Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::2]:2747","req_id":"ef91d7c6-0e56-43bb-abb1-4bd2447c759e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 51229,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:42:04.452871789Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::2]:56909","req_id":"3ded5e1c-b3ce-4b5f-9741-d1b174e6a7df","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:04.508294951Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:3::2]:36100"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:04.50904879Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"new_generation":1,"req_id":"c43c736c-1308-401e-869c-c132b123cb45","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:04.509071441Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:04.521385751Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:04.521414942Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"new_generation":1,"req_id":"c43c736c-1308-401e-869c-c132b123cb45","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:42:04.521441862Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"uri":"/config","method":"PUT","req_id":"c43c736c-1308-401e-869c-c132b123cb45","remote_addr":"[fd00:1122:3344:3::2]:36100","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":12757,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 8056,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:42:15.416543568Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::1b]:63058","req_id":"2e9d1753-3b05-4e5a-ad89-0a243a49f895","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 6166,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:42:15.42084966Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::1b]:41712","req_id":"7b883369-c804-4522-bf3c-3336466e9c8f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 56568,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:42:15.424885015Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::1b]:41814","req_id":"407b216e-5b1c-491e-9fda-4fcd9771f0a0","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 22224,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:31.881059621Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::d]:58651","req_id":"f27e5001-1cbc-4741-9064-8d966e340767","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 19270,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:41.867140378Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::d]:64960","req_id":"324696b1-a7c2-4bf5-8ee0-0fd6ee500d52","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 32312,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:42.178038747Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::c]:11659","req_id":"a50613fe-7432-41e6-aa0e-2a7728eb2e44","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 24427,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:43.431396595Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::b]:9568","req_id":"a1750dde-1f58-4680-b90f-73db49ae7293","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 26858,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:44.691284813Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::a]:36288","req_id":"588560ed-8df4-47fb-ba0b-3dea4e96c60a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 10663,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 54,\n 52,\n 53,\n 99,\n 53,\n 102,\n 100,\n 102,\n 45,\n 53,\n 55,\n 56,\n 98,\n 45,\n 52,\n 97,\n 56,\n 54,\n 45,\n 56,\n 52,\n 57,\n 100,\n 45,\n 48,\n 57,\n 49,\n 55,\n 54,\n 50,\n 100,\n 102,\n 48,\n 55,\n 53,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:45.898218854Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::e]:36028","req_id":"bb8a2775-9867-4f79-8f85-4ce7c1938c25","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 10663,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 54,\n 52,\n 53,\n 99,\n 53,\n 102,\n 100,\n 102,\n 45,\n 53,\n 55,\n 56,\n 98,\n 45,\n 52,\n 97,\n 56,\n 54,\n 45,\n 56,\n 52,\n 57,\n 100,\n 45,\n 48,\n 57,\n 49,\n 55,\n 54,\n 50,\n 100,\n 102,\n 48,\n 55,\n 53,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:45.898982394Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::e]:64819","req_id":"cdc492f7-30cb-47d9-be98-32e85c2a908b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40853,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:43:51.868377369Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::d]:56502","req_id":"f02be065-52eb-4d4b-a44f-56d348b8ff05","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18237,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:01.871364576Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::d]:54728","req_id":"e04da854-58a9-48cc-8b99-782cd433f155","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64931,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:11.873735579Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::d]:58297","req_id":"ce430433-3507-40a3-a928-679da9bf8a01","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 44783,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:21.875421742Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::d]:59290","req_id":"a941f2df-9a6b-4788-9878-2c93f6321190","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 12769,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:23.11490624Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::c]:29775","req_id":"b59b55f1-50b4-44c7-875a-4375b6e6bb77","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:23.357461861Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:41381"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:23.418441767Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"uri":"/config","method":"GET","req_id":"281a69f5-4a6e-404e-93d9-ee0e311e0103","remote_addr":"[fd00:1122:3344:101::c]:41381","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":38924,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:26.631915732Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:65440"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:26.643123684Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"new_generation":1,"req_id":"4d2e5457-f635-40d8-a0af-ac838fe84b20","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:26.643195326Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"new_generation":1,"req_id":"4d2e5457-f635-40d8-a0af-ac838fe84b20","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:26.643205916Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"uri":"/config","method":"PUT","req_id":"4d2e5457-f635-40d8-a0af-ac838fe84b20","remote_addr":"[fd00:1122:3344:101::c]:65440","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":9285,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41506,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:26.749767909Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::c]:59142","req_id":"28cf61f1-ed7c-4990-8646-2708e0f387dc","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:26.863178039Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:40862"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:26.863471866Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"uri":"/config","method":"GET","req_id":"dcf5dabd-e2d4-4afa-ba59-82c40fcc4723","remote_addr":"[fd00:1122:3344:101::c]:40862","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":234,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 32622,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:27.455390567Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::c]:56645","req_id":"9f8bd99b-8e38-4369-9147-3b84542d21b3","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:27.576013735Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:47652"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:27.576265812Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"uri":"/config","method":"GET","req_id":"d89a74e8-8c8b-4847-a0ba-ee7f5efe4089","remote_addr":"[fd00:1122:3344:101::c]:47652","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":212,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 29856,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:31.87724449Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::d]:52026","req_id":"5201d2b4-2c62-4366-8bde-61b5b359207c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 27672,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:34.150054802Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::b]:35630","req_id":"2c67b533-fcd2-4f26-8ca8-11b03132c4c0","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:34.264376786Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:61122"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:34.264664974Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"uri":"/config","method":"GET","req_id":"8416005a-24eb-4166-be6a-3996e9bd33c4","remote_addr":"[fd00:1122:3344:101::b]:61122","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":232,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 12565,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:35.502874388Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::a]:8925","req_id":"cbc265e0-c5d0-4056-8933-296e6b9ce699","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:35.60825455Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:34098"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:35.608583458Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"uri":"/config","method":"GET","req_id":"10b49a30-1351-480e-abb4-b72b37de63a4","remote_addr":"[fd00:1122:3344:101::a]:34098","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":250,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 32641,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:41.879410206Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::d]:53049","req_id":"d9098742-e8e0-4014-bb8e-197b7152135d","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:41.918235086Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:55715"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:41.921686206Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"new_generation":1,"req_id":"b24059cb-7b18-44ce-9e68-d15e618938ae","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:41.921752588Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"new_generation":1,"req_id":"b24059cb-7b18-44ce-9e68-d15e618938ae","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:41.921765228Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"uri":"/config","method":"PUT","req_id":"b24059cb-7b18-44ce-9e68-d15e618938ae","remote_addr":"[fd00:1122:3344:101::c]:55715","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":327,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 44512,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:42.83663841Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::c]:58532","req_id":"ae39cc21-5f6e-43ac-b932-ee61ddc019c3","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:42.935993625Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:45605"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:42.936268282Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"uri":"/config","method":"GET","req_id":"ead6dcaf-e51e-40e2-b7ce-c208ebbd3264","remote_addr":"[fd00:1122:3344:101::c]:45605","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":215,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:43.431440705Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:61956"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:43.434813993Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"new_generation":1,"req_id":"870d5f3a-e009-443b-9400-29cce2ce0275","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:43.434857234Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"new_generation":1,"req_id":"870d5f3a-e009-443b-9400-29cce2ce0275","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:43.434873305Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"uri":"/config","method":"PUT","req_id":"870d5f3a-e009-443b-9400-29cce2ce0275","remote_addr":"[fd00:1122:3344:101::b]:61956","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":207,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48000,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:44.41942097Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::b]:28566","req_id":"0fb8e9fa-175a-43c3-8e88-fe75933dca9f","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:44.516966478Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:36585"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:44.517216484Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"uri":"/config","method":"GET","req_id":"85ff2a4e-1412-44e0-80e7-95338030aa56","remote_addr":"[fd00:1122:3344:101::b]:36585","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":192,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:44.815241408Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:54565"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:44.818340669Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"new_generation":1,"req_id":"6477e35d-6060-47ff-a9c4-9fcb8e7977ca","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:44.81837827Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"new_generation":1,"req_id":"6477e35d-6060-47ff-a9c4-9fcb8e7977ca","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:44.81841045Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"uri":"/config","method":"PUT","req_id":"6477e35d-6060-47ff-a9c4-9fcb8e7977ca","remote_addr":"[fd00:1122:3344:101::a]:54565","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":261,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 25985,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:45.636214337Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::a]:34453","req_id":"b5e67fc6-11b3-4e09-bfe8-f4d09e9c6280","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:45.732460431Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:37382"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-24T21:44:45.732736758Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"uri":"/config","method":"GET","req_id":"3b25b9ae-a4ac-4f97-b000-ad9fb9672bd4","remote_addr":"[fd00:1122:3344:101::a]:37382","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":223,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64840,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:44:51.88072945Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::d]:58023","req_id":"0ea933a8-e62a-485d-9fc7-c4db8cd2a454","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40945,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_e9266dde-d155-4ae7-915c-8af8a7eb59ad.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_e9266dde-d155-4ae7-915c-8af8a7eb59ad.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 101,\n 57,\n 50,\n 54,\n 54,\n 100,\n 100,\n 101,\n 45,\n 100,\n 49,\n 53,\n 53,\n 45,\n 52,\n 97,\n 101,\n 55,\n 45,\n 57,\n 49,\n 53,\n 99,\n 45,\n 56,\n 97,\n 102,\n 56,\n 97,\n 55,\n 101,\n 98,\n 53,\n 57,\n 97,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_e9266dde-d155-4ae7-915c-8af8a7eb59ad.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.028647493Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::4]:47453","req_id":"920ebd02-3aee-4171-87e4-48fc70d2a5b7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40945,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_e9266dde-d155-4ae7-915c-8af8a7eb59ad.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_e9266dde-d155-4ae7-915c-8af8a7eb59ad.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 101,\n 57,\n 50,\n 54,\n 54,\n 100,\n 100,\n 101,\n 45,\n 100,\n 49,\n 53,\n 53,\n 45,\n 52,\n 97,\n 101,\n 55,\n 45,\n 57,\n 49,\n 53,\n 99,\n 45,\n 56,\n 97,\n 102,\n 56,\n 97,\n 55,\n 101,\n 98,\n 53,\n 57,\n 97,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_e9266dde-d155-4ae7-915c-8af8a7eb59ad.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.029427954Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::4]:45701","req_id":"6a5dd412-1ad2-4c84-a937-b1fc63b06244","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62078,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_4e16d07e-4a1c-4ea5-8a0a-2d13cf643967.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_4e16d07e-4a1c-4ea5-8a0a-2d13cf643967.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 52,\n 101,\n 49,\n 54,\n 100,\n 48,\n 55,\n 101,\n 45,\n 52,\n 97,\n 49,\n 99,\n 45,\n 52,\n 101,\n 97,\n 53,\n 45,\n 56,\n 97,\n 48,\n 97,\n 45,\n 50,\n 100,\n 49,\n 51,\n 99,\n 102,\n 54,\n 52,\n 51,\n 57,\n 54,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_4e16d07e-4a1c-4ea5-8a0a-2d13cf643967.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.070159784Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::5]:61949","req_id":"e6424b9f-f3ee-4054-8acc-60e1937ec6ca","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62078,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_4e16d07e-4a1c-4ea5-8a0a-2d13cf643967.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_4e16d07e-4a1c-4ea5-8a0a-2d13cf643967.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 52,\n 101,\n 49,\n 54,\n 100,\n 48,\n 55,\n 101,\n 45,\n 52,\n 97,\n 49,\n 99,\n 45,\n 52,\n 101,\n 97,\n 53,\n 45,\n 56,\n 97,\n 48,\n 97,\n 45,\n 50,\n 100,\n 49,\n 51,\n 99,\n 102,\n 54,\n 52,\n 51,\n 57,\n 54,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_4e16d07e-4a1c-4ea5-8a0a-2d13cf643967.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.070628436Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::5]:58201","req_id":"e5aeb478-10bd-4448-aea1-af46f9a31a7e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52291,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 54,\n 52,\n 53,\n 99,\n 53,\n 102,\n 100,\n 102,\n 45,\n 53,\n 55,\n 56,\n 98,\n 45,\n 52,\n 97,\n 56,\n 54,\n 45,\n 56,\n 52,\n 57,\n 100,\n 45,\n 48,\n 57,\n 49,\n 55,\n 54,\n 50,\n 100,\n 102,\n 48,\n 55,\n 53,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.122644139Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::e]:34534","req_id":"c8b385ac-3446-4646-97c9-bd72d41357b7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52291,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 54,\n 52,\n 53,\n 99,\n 53,\n 102,\n 100,\n 102,\n 45,\n 53,\n 55,\n 56,\n 98,\n 45,\n 52,\n 97,\n 56,\n 54,\n 45,\n 56,\n 52,\n 57,\n 100,\n 45,\n 48,\n 57,\n 49,\n 55,\n 54,\n 50,\n 100,\n 102,\n 48,\n 55,\n 53,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_645c5fdf-578b-4a86-849d-091762df0757.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.123181263Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::e]:49905","req_id":"05d3f3ce-a065-472e-96b9-503b46792c6c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48816,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_ab61c6f0-8d7c-41ed-b2c9-5f8c105d1767.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_ab61c6f0-8d7c-41ed-b2c9-5f8c105d1767.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 97,\n 98,\n 54,\n 49,\n 99,\n 54,\n 102,\n 48,\n 45,\n 56,\n 100,\n 55,\n 99,\n 45,\n 52,\n 49,\n 101,\n 100,\n 45,\n 98,\n 50,\n 99,\n 57,\n 45,\n 53,\n 102,\n 56,\n 99,\n 49,\n 48,\n 53,\n 100,\n 49,\n 55,\n 54,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_ab61c6f0-8d7c-41ed-b2c9-5f8c105d1767.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.441194237Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::1b]:54779","req_id":"c5271d27-ab9e-4c21-bfab-b022254d6ca2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 47032,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_b18bd539-4d7c-41b4-93ac-99b66585e432.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_b18bd539-4d7c-41b4-93ac-99b66585e432.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 49,\n 56,\n 98,\n 100,\n 53,\n 51,\n 57,\n 45,\n 52,\n 100,\n 55,\n 99,\n 45,\n 52,\n 49,\n 98,\n 52,\n 45,\n 57,\n 51,\n 97,\n 99,\n 45,\n 57,\n 57,\n 98,\n 54,\n 54,\n 53,\n 56,\n 53,\n 101,\n 52,\n 51,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_b18bd539-4d7c-41b4-93ac-99b66585e432.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.781041639Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::6]:40623","req_id":"04b4d8bb-c6e6-4590-b712-e43d6deebe58","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 47032,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_b18bd539-4d7c-41b4-93ac-99b66585e432.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_b18bd539-4d7c-41b4-93ac-99b66585e432.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 49,\n 56,\n 98,\n 100,\n 53,\n 51,\n 57,\n 45,\n 52,\n 100,\n 55,\n 99,\n 45,\n 52,\n 49,\n 98,\n 52,\n 45,\n 57,\n 51,\n 97,\n 99,\n 45,\n 57,\n 57,\n 98,\n 54,\n 54,\n 53,\n 56,\n 53,\n 101,\n 52,\n 51,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_b18bd539-4d7c-41b4-93ac-99b66585e432.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.781813229Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::6]:50497","req_id":"3c8ab123-0fc5-41ed-a4a1-d8a8715be71c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57674,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_c53bb4cd-f9ba-4262-a310-9aab9e790d29.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_c53bb4cd-f9ba-4262-a310-9aab9e790d29.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 99,\n 53,\n 51,\n 98,\n 98,\n 52,\n 99,\n 100,\n 45,\n 102,\n 57,\n 98,\n 97,\n 45,\n 52,\n 50,\n 54,\n 50,\n 45,\n 97,\n 51,\n 49,\n 48,\n 45,\n 57,\n 97,\n 97,\n 98,\n 57,\n 101,\n 55,\n 57,\n 48,\n 100,\n 50,\n 57,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_c53bb4cd-f9ba-4262-a310-9aab9e790d29.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.801509441Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::7]:44946","req_id":"7fdc8474-df33-4155-ba67-a040f6349c1a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57674,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_c53bb4cd-f9ba-4262-a310-9aab9e790d29.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_c53bb4cd-f9ba-4262-a310-9aab9e790d29.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 99,\n 53,\n 51,\n 98,\n 98,\n 52,\n 99,\n 100,\n 45,\n 102,\n 57,\n 98,\n 97,\n 45,\n 52,\n 50,\n 54,\n 50,\n 45,\n 97,\n 51,\n 49,\n 48,\n 45,\n 57,\n 97,\n 97,\n 98,\n 57,\n 101,\n 55,\n 57,\n 48,\n 100,\n 50,\n 57,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_c53bb4cd-f9ba-4262-a310-9aab9e790d29.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.802122907Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::7]:34938","req_id":"58d0890d-d044-497a-a5ab-e5b612857bc7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50659,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.891609384Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::2]:61482","req_id":"96957d20-8f7f-4be3-9941-e49e0c1eef7b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50659,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:00.892255861Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::2]:54233","req_id":"ed0ded78-1384-469b-bd2b-004f749dbfbe","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64365,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_81c0d863-a7d9-422c-93e2-a5815214b3d5.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_81c0d863-a7d9-422c-93e2-a5815214b3d5.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 49,\n 99,\n 48,\n 100,\n 56,\n 54,\n 51,\n 45,\n 97,\n 55,\n 100,\n 57,\n 45,\n 52,\n 50,\n 50,\n 99,\n 45,\n 57,\n 51,\n 101,\n 50,\n 45,\n 97,\n 53,\n 56,\n 49,\n 53,\n 50,\n 49,\n 52,\n 98,\n 51,\n 100,\n 53,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_81c0d863-a7d9-422c-93e2-a5815214b3d5.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:01.010492368Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::3]:33357","req_id":"c1a58300-4ca5-4895-94fc-3f152e919259","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64365,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_81c0d863-a7d9-422c-93e2-a5815214b3d5.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_81c0d863-a7d9-422c-93e2-a5815214b3d5.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 49,\n 99,\n 48,\n 100,\n 56,\n 54,\n 51,\n 45,\n 97,\n 55,\n 100,\n 57,\n 45,\n 52,\n 50,\n 50,\n 99,\n 45,\n 57,\n 51,\n 101,\n 50,\n 45,\n 97,\n 53,\n 56,\n 49,\n 53,\n 50,\n 49,\n 52,\n 98,\n 51,\n 100,\n 53,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_81c0d863-a7d9-422c-93e2-a5815214b3d5.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:01.011093343Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::3]:65321","req_id":"89699c2b-8ae6-44b4-8f1d-0665541a785d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 37416,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-24T21:45:01.881733615Z","hostname":"oxz_internal_dns_01b61f0d-4efe-49b0-b33b-5ce793d0956d","pid":5456,"peer_addr":"[fd00:1122:3344:101::d]:59961","req_id":"0a0fed16-69e2-4668-a927-87e8903ea46b","component":"dns"}