[ Mar 24 20:54:37 Enabled. ] [ Mar 24 20:54:37 Rereading configuration. ] [ Mar 24 20:54:37 Rereading configuration. ] [ Mar 24 20:54:38 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:2::1]:5353 --dns-address [fd00:1122:3344:2::1]:53 &"). ] [ Mar 24 20:54:38 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:54:38.879575573Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"dns_config":"Config { bind_address: [fd00:1122:3344:2::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:54:38.879717467Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:54:38.90430543Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:54:38.90432276Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:54:38.904367922Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"component":"dns","local_address":"[fd00:1122:3344:2::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:54:38.904849715Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:54:39.237678478Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:2::2]:52945"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:54:39.238431409Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"new_generation":1,"req_id":"8ba6fb1a-f2a9-433a-874e-4ddc34f894dc","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:54:39.238445849Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:54:39.255994022Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:54:39.256017973Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"new_generation":1,"req_id":"8ba6fb1a-f2a9-433a-874e-4ddc34f894dc","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:54:39.256029503Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"PUT","req_id":"8ba6fb1a-f2a9-433a-874e-4ddc34f894dc","remote_addr":"[fd00:1122:3344:2::2]:52945","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":17971,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57253,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:54:51.422818589Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::1b]:33342","req_id":"6192897d-5a95-4861-b345-584ab64a940b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50730,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:54:51.427057033Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::1b]:35879","req_id":"af76857f-24f9-4796-be65-ccffbdc05ed6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23688,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:54:51.431120432Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::1b]:42644","req_id":"8aa7e992-0ec3-4cf1-92c7-81375b2b2964","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1165,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:56:14.492152943Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::1b]:54099","req_id":"a083db17-9f25-4181-a858-6a2a207006af","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 15974,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:56:14.494587219Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::1b]:58686","req_id":"15ac3325-ba26-4fab-9e87-190db33a6241","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 7041,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:56:14.494608289Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::1b]:48208","req_id":"5eaf43c4-fd87-4653-a25f-9be9fe90c1cd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64657,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:56:55.453407678Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:50293","req_id":"2ced9e31-d39c-4b06-83c5-2fb170996794","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 27998,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:57:03.762846453Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::1b]:39734","req_id":"c5dd6860-f9cb-4525-9df1-9ce93a335405","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 44247,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:57:03.773860908Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::1b]:42275","req_id":"34803340-2955-441a-a037-70c3c2ada41a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 25988,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:57:03.778367291Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::1b]:53408","req_id":"9a762758-efe7-4af4-92c6-11f9be509627","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 60511,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:57:05.45650677Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:58616","req_id":"11340a85-d9b4-4354-bff5-06ec8ae5509a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1209,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:57:05.542692634Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::c]:6725","req_id":"46599a03-fc26-4b14-81dc-db2f625132e1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 25557,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:57:05.652627799Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::b]:26497","req_id":"df44224e-96aa-4d25-8e11-96c7c00e798b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 29794,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:57:05.786294003Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::a]:42479","req_id":"025468d9-ae54-415d-8b3b-9ec27f58f927","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3118,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_7cb4efe7-1c84-48e5-84af-f4ad7406d853.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_7cb4efe7-1c84-48e5-84af-f4ad7406d853.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 55,\n 99,\n 98,\n 52,\n 101,\n 102,\n 101,\n 55,\n 45,\n 49,\n 99,\n 56,\n 52,\n 45,\n 52,\n 56,\n 101,\n 53,\n 45,\n 56,\n 52,\n 97,\n 102,\n 45,\n 102,\n 52,\n 97,\n 100,\n 55,\n 52,\n 48,\n 54,\n 100,\n 56,\n 53,\n 51,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_7cb4efe7-1c84-48e5-84af-f4ad7406d853.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:57:06.76008929Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::e]:44550","req_id":"31eff074-ac8d-4482-9ddd-bc470ac56c7a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3118,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_7cb4efe7-1c84-48e5-84af-f4ad7406d853.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_7cb4efe7-1c84-48e5-84af-f4ad7406d853.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 55,\n 99,\n 98,\n 52,\n 101,\n 102,\n 101,\n 55,\n 45,\n 49,\n 99,\n 56,\n 52,\n 45,\n 52,\n 56,\n 101,\n 53,\n 45,\n 56,\n 52,\n 97,\n 102,\n 45,\n 102,\n 52,\n 97,\n 100,\n 55,\n 52,\n 48,\n 54,\n 100,\n 56,\n 53,\n 51,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_7cb4efe7-1c84-48e5-84af-f4ad7406d853.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:57:06.760771548Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::e]:47396","req_id":"ed857f82-c984-42e7-a895-53ec15ae634f","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:57:07.26187944Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:56161"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:57:07.321637662Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"new_generation":1,"req_id":"4517b4b9-c0d1-4e67-95be-ad73db732854","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:57:07.321714684Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"new_generation":1,"req_id":"4517b4b9-c0d1-4e67-95be-ad73db732854","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:57:07.323696147Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"PUT","req_id":"4517b4b9-c0d1-4e67-95be-ad73db732854","remote_addr":"[fd00:1122:3344:101::c]:56161","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":24946,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:57:09.038236227Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:48507"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:57:09.03833008Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:59111"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:57:09.03834025Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:49364"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:57:09.049244134Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"GET","req_id":"f44436a8-8ebf-41d0-90cf-02900087c965","remote_addr":"[fd00:1122:3344:101::b]:59111","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":10876,"response_code":200} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:57:09.049299626Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"GET","req_id":"b566b73e-149c-4bb1-8417-b04720f6ac3e","remote_addr":"[fd00:1122:3344:101::c]:49364","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":10275,"response_code":200} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:57:09.049331306Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"GET","req_id":"2a546ae2-4d17-46c5-af8b-ec26bee830e8","remote_addr":"[fd00:1122:3344:101::a]:48507","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":10911,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42228,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:57:09.595974686Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::b]:4423","req_id":"793cc984-79f8-4b0d-a09e-ceac8753000c","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:57:09.713235388Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:38923"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:57:09.713484995Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"GET","req_id":"27723708-6bee-4e0b-90a1-35c8b58b144d","remote_addr":"[fd00:1122:3344:101::b]:38923","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":201,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41144,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:57:10.493161838Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::c]:18496","req_id":"3da875ae-76f0-419e-a8a7-346ef32ceb77","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:57:10.718995725Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:36539"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:57:10.719228896Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"GET","req_id":"279173e9-3729-4d34-ba01-799242ed7ec1","remote_addr":"[fd00:1122:3344:101::c]:36539","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":195,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3063,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:57:11.234330787Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::c]:3961","req_id":"0c70269a-5bce-4a6b-9961-19f53774c944","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:57:11.406744796Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:44881"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:57:11.407130866Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"GET","req_id":"b7e3a9d0-f87c-41c7-bec4-e4d51790b770","remote_addr":"[fd00:1122:3344:101::c]:44881","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":259,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 36842,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:57:15.459800986Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:61040","req_id":"9cb7b4a1-4462-4043-9d34-de2bf01f3c3b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 9354,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:57:16.333235049Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::b]:44131","req_id":"37a59a56-b15a-4d56-947f-76e02185a2f7","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:57:16.443612244Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:47338"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:57:16.443844811Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"GET","req_id":"9b92f715-cee9-4211-86e9-51f90dc3e82f","remote_addr":"[fd00:1122:3344:101::b]:47338","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":193,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31665,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:57:16.446636546Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::a]:60827","req_id":"fbd24510-5ad5-4e34-9845-3863eaec6fd6","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:57:16.563437676Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:52747"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:57:16.563728064Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"GET","req_id":"0425eda7-6739-431a-a33e-bdfbf786fab9","remote_addr":"[fd00:1122:3344:101::a]:52747","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":244,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31530,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:57:25.461624924Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:59328","req_id":"9fb01085-8028-45b4-a9f0-2f75ad35fc77","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21436,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:57:35.463486869Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:50731","req_id":"466012cc-6cc2-4fb8-be9d-5963d8a36b12","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 15878,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:57:45.464400046Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:65281","req_id":"ed3b1bd0-d705-424c-a12e-0add734aeecb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 33798,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:57:55.465797795Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:59519","req_id":"79121dbe-636f-45cd-abb8-5f27785350df","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:58:05.294177002Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:60525"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:58:05.29446604Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"new_generation":1,"req_id":"d5cf5d2a-dd4f-4baa-bc10-ee72bc7bf031","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:58:05.294499001Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"new_generation":1,"req_id":"d5cf5d2a-dd4f-4baa-bc10-ee72bc7bf031","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:58:05.294517531Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"PUT","req_id":"d5cf5d2a-dd4f-4baa-bc10-ee72bc7bf031","remote_addr":"[fd00:1122:3344:101::c]:60525","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":232,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 19416,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:58:05.46734169Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:63735","req_id":"a00f7235-c33a-4700-860d-5d3703947b6c","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:58:05.526542907Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:60340"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:58:05.527093492Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"new_generation":1,"req_id":"4e0b54f5-7922-430c-adb2-878845a77abe","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:58:05.527124783Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"new_generation":1,"req_id":"4e0b54f5-7922-430c-adb2-878845a77abe","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:58:05.527159574Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"PUT","req_id":"4e0b54f5-7922-430c-adb2-878845a77abe","remote_addr":"[fd00:1122:3344:101::a]:60340","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":423,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:58:05.881605341Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:44445"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:58:05.88194492Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"new_generation":1,"req_id":"ecefaa05-309e-4b71-be06-dfee1ec458fc","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:58:05.881980781Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"new_generation":1,"req_id":"ecefaa05-309e-4b71-be06-dfee1ec458fc","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:58:05.881988162Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"PUT","req_id":"ecefaa05-309e-4b71-be06-dfee1ec458fc","remote_addr":"[fd00:1122:3344:101::b]:44445","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":298,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 33548,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:58:06.85804152Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::a]:29579","req_id":"c6a23425-bf55-4f86-92e4-19c53fae6af2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43801,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:58:06.864310899Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::b]:6855","req_id":"5515c2eb-f753-4d5f-866a-cf747910319e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23354,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:58:06.86732983Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::c]:16981","req_id":"770a48b4-4c7a-4cdf-9825-932b3593c6b2","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:58:06.967585603Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:33953"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:58:06.967863941Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"GET","req_id":"d35af30d-580f-4c6b-93be-9aeeb9af36df","remote_addr":"[fd00:1122:3344:101::a]:33953","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":230,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:58:07.007259923Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:34601"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:58:07.007564821Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"GET","req_id":"b8d0ee20-19f8-4458-9eeb-b10829ecee02","remote_addr":"[fd00:1122:3344:101::b]:34601","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":254,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:58:07.042665248Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:52291"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:58:07.042888884Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"GET","req_id":"74c35d89-5457-48a5-b67f-3f76efa52625","remote_addr":"[fd00:1122:3344:101::c]:52291","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":187,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 26370,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:58:15.468258847Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:60716","req_id":"45b2af17-3d0d-4bba-a778-cb98f8fedad0","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 37120,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:58:25.469517077Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:55554","req_id":"f134db8e-65a5-4bf7-9992-d565776d7931","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1301,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:58:35.471273232Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:62005","req_id":"0393513f-4eb1-45d3-a3d2-6b36935517ca","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 27251,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:58:45.472074949Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:63158","req_id":"0a59f52f-0e35-46ae-a4dc-1c0a4a8dd981","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 49542,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:58:50.63865665Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::1b]:46715","req_id":"423f8449-2a16-434b-a316-d9f0a654069a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 9838,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:58:50.642807522Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::1b]:38899","req_id":"7fe29a0d-f7d5-4145-8b7b-7a237ccd4b92","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59067,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:58:50.646784Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::1b]:63682","req_id":"29a4b52a-d1a1-4d07-b84e-88681bc6bf06","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18334,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:58:55.473510116Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:61280","req_id":"b9f9353f-c07f-422f-9b01-70499471163f","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:59:05.294383593Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:34054"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:59:05.294611449Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"new_generation":1,"req_id":"710706fb-c2c5-4b58-ac62-0cf15bbe1766","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:59:05.29463691Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"new_generation":1,"req_id":"710706fb-c2c5-4b58-ac62-0cf15bbe1766","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:59:05.29464443Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"PUT","req_id":"710706fb-c2c5-4b58-ac62-0cf15bbe1766","remote_addr":"[fd00:1122:3344:101::c]:34054","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":153,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53915,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:59:05.474251763Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:64581","req_id":"4d9f9578-fbca-4362-9ba9-b1c948036269","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:59:05.484097858Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:45048"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:59:05.484369575Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"new_generation":1,"req_id":"ea8abfc5-68a2-4f90-a4ba-8b35af4c08ac","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:59:05.484410966Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"new_generation":1,"req_id":"ea8abfc5-68a2-4f90-a4ba-8b35af4c08ac","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:59:05.484421747Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"PUT","req_id":"ea8abfc5-68a2-4f90-a4ba-8b35af4c08ac","remote_addr":"[fd00:1122:3344:101::a]:45048","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":193,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:59:05.525230617Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:40591"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:59:05.52534469Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"new_generation":1,"req_id":"cce5dfed-adf0-4417-b938-8f89c067c2cf","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:59:05.525358321Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"new_generation":1,"req_id":"cce5dfed-adf0-4417-b938-8f89c067c2cf","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:59:05.525386401Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"PUT","req_id":"cce5dfed-adf0-4417-b938-8f89c067c2cf","remote_addr":"[fd00:1122:3344:101::b]:40591","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":230,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 60343,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:59:09.298213122Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::c]:65222","req_id":"0484beba-81e5-420e-b5c5-2d371429ef06","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14600,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:59:09.303867494Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::a]:24414","req_id":"4722cf93-3a30-4b29-923f-00905e21153e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41663,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:59:09.303998638Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::b]:39616","req_id":"379f89fb-ba47-43b5-9861-215fbea7cae5","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:59:09.404111426Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:48894"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:59:09.404329992Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"GET","req_id":"56545531-93c9-491c-8a18-4cfde72e8c52","remote_addr":"[fd00:1122:3344:101::b]:48894","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":180,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:59:09.443717104Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:61799"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:59:09.443981571Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"GET","req_id":"fc9907b0-7cc3-46c8-a92c-b5d380ec85df","remote_addr":"[fd00:1122:3344:101::a]:61799","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":228,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:59:09.493714272Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:44147"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T20:59:09.493923458Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"uri":"/config","method":"GET","req_id":"3b0d231d-81be-46fe-87d0-8f864c2bb5a7","remote_addr":"[fd00:1122:3344:101::c]:44147","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":178,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18828,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:59:15.476447446Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:63425","req_id":"abd25130-d80b-40dd-9c70-7bccd65911cc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42556,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:59:25.478858668Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:55159","req_id":"0bb9c9e4-ad38-4b59-ab03-a72b9575d9bd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 44339,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:59:35.479562277Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:54111","req_id":"3504e5df-28b3-4c34-a0e2-4692b998fc78","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 15635,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:59:45.481016868Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:61285","req_id":"ba01f6f1-fe5e-41d0-a3b4-f0888cc97fad","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 32857,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T20:59:55.483148513Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::d]:55478","req_id":"50f33c4b-c094-4bfa-a887-53139cc6cd49","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 32350,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T21:00:00.276576974Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::2]:62274","req_id":"6f5e8a8a-f229-40af-be6d-90b86881a936","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 32350,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T21:00:00.277047287Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::2]:34024","req_id":"f9efb6e1-1110-47a3-9e77-ca0cfa4de136","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 45715,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_6b02d2eb-4373-4f12-8efc-4c0e3f4fa77b.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_6b02d2eb-4373-4f12-8efc-4c0e3f4fa77b.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 98,\n 48,\n 50,\n 100,\n 50,\n 101,\n 98,\n 45,\n 52,\n 51,\n 55,\n 51,\n 45,\n 52,\n 102,\n 49,\n 50,\n 45,\n 56,\n 101,\n 102,\n 99,\n 45,\n 52,\n 99,\n 48,\n 101,\n 51,\n 102,\n 52,\n 102,\n 97,\n 55,\n 55,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_6b02d2eb-4373-4f12-8efc-4c0e3f4fa77b.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T21:00:00.307846488Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::3]:44786","req_id":"6bdb28dd-bec7-449a-ac05-7e4b58b24dc6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 45715,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_6b02d2eb-4373-4f12-8efc-4c0e3f4fa77b.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_6b02d2eb-4373-4f12-8efc-4c0e3f4fa77b.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 98,\n 48,\n 50,\n 100,\n 50,\n 101,\n 98,\n 45,\n 52,\n 51,\n 55,\n 51,\n 45,\n 52,\n 102,\n 49,\n 50,\n 45,\n 56,\n 101,\n 102,\n 99,\n 45,\n 52,\n 99,\n 48,\n 101,\n 51,\n 102,\n 52,\n 102,\n 97,\n 55,\n 55,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_6b02d2eb-4373-4f12-8efc-4c0e3f4fa77b.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T21:00:00.308265469Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::3]:45380","req_id":"89ae31e0-75c1-4dd9-930f-b6c899ec1813","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 15303,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_1345db7a-abc1-4a4c-a49c-e6c56102e521.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_1345db7a-abc1-4a4c-a49c-e6c56102e521.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 49,\n 51,\n 52,\n 53,\n 100,\n 98,\n 55,\n 97,\n 45,\n 97,\n 98,\n 99,\n 49,\n 45,\n 52,\n 97,\n 52,\n 99,\n 45,\n 97,\n 52,\n 57,\n 99,\n 45,\n 101,\n 54,\n 99,\n 53,\n 54,\n 49,\n 48,\n 50,\n 101,\n 53,\n 50,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_1345db7a-abc1-4a4c-a49c-e6c56102e521.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T21:00:00.371768001Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::7]:33710","req_id":"48593146-292d-476e-8651-3bf8c642ae7c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 15303,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_1345db7a-abc1-4a4c-a49c-e6c56102e521.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_1345db7a-abc1-4a4c-a49c-e6c56102e521.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 49,\n 51,\n 52,\n 53,\n 100,\n 98,\n 55,\n 97,\n 45,\n 97,\n 98,\n 99,\n 49,\n 45,\n 52,\n 97,\n 52,\n 99,\n 45,\n 97,\n 52,\n 57,\n 99,\n 45,\n 101,\n 54,\n 99,\n 53,\n 54,\n 49,\n 48,\n 50,\n 101,\n 53,\n 50,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_1345db7a-abc1-4a4c-a49c-e6c56102e521.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T21:00:00.372336777Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::7]:43844","req_id":"e220315c-d709-47b5-b31f-2104333aa033","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 45318,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_c3b068bb-6f9c-4c4e-b559-e3154ac75f47.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_c3b068bb-6f9c-4c4e-b559-e3154ac75f47.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 99,\n 51,\n 98,\n 48,\n 54,\n 56,\n 98,\n 98,\n 45,\n 54,\n 102,\n 57,\n 99,\n 45,\n 52,\n 99,\n 52,\n 101,\n 45,\n 98,\n 53,\n 53,\n 57,\n 45,\n 101,\n 51,\n 49,\n 53,\n 52,\n 97,\n 99,\n 55,\n 53,\n 102,\n 52,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_c3b068bb-6f9c-4c4e-b559-e3154ac75f47.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T21:00:00.555513496Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::6]:50102","req_id":"c74b86b9-1a52-484f-a5a7-407974273e25","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 45318,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_c3b068bb-6f9c-4c4e-b559-e3154ac75f47.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_c3b068bb-6f9c-4c4e-b559-e3154ac75f47.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 99,\n 51,\n 98,\n 48,\n 54,\n 56,\n 98,\n 98,\n 45,\n 54,\n 102,\n 57,\n 99,\n 45,\n 52,\n 99,\n 52,\n 101,\n 45,\n 98,\n 53,\n 53,\n 57,\n 45,\n 101,\n 51,\n 49,\n 53,\n 52,\n 97,\n 99,\n 55,\n 53,\n 102,\n 52,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_c3b068bb-6f9c-4c4e-b559-e3154ac75f47.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T21:00:00.55605003Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::6]:33339","req_id":"8d8f5553-038d-4401-94bf-b8ffbf2eea85","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57585,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_1920db37-87f5-4a96-b3ef-1db2eb59e9a5.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_1920db37-87f5-4a96-b3ef-1db2eb59e9a5.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 49,\n 57,\n 50,\n 48,\n 100,\n 98,\n 51,\n 55,\n 45,\n 56,\n 55,\n 102,\n 53,\n 45,\n 52,\n 97,\n 57,\n 54,\n 45,\n 98,\n 51,\n 101,\n 102,\n 45,\n 49,\n 100,\n 98,\n 50,\n 101,\n 98,\n 53,\n 57,\n 101,\n 57,\n 97,\n 53,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_1920db37-87f5-4a96-b3ef-1db2eb59e9a5.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T21:00:00.57678996Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::5]:44628","req_id":"c9406112-bae7-4ebd-bdec-64a78ea29fc4","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57585,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_1920db37-87f5-4a96-b3ef-1db2eb59e9a5.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_1920db37-87f5-4a96-b3ef-1db2eb59e9a5.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 49,\n 57,\n 50,\n 48,\n 100,\n 98,\n 51,\n 55,\n 45,\n 56,\n 55,\n 102,\n 53,\n 45,\n 52,\n 97,\n 57,\n 54,\n 45,\n 98,\n 51,\n 101,\n 102,\n 45,\n 49,\n 100,\n 98,\n 50,\n 101,\n 98,\n 53,\n 57,\n 101,\n 57,\n 97,\n 53,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_1920db37-87f5-4a96-b3ef-1db2eb59e9a5.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T21:00:00.577253722Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::5]:65017","req_id":"1d3797f0-e98b-486d-bcd3-3ddd0ed9e1a6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50316,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_709c4704-f768-4640-a198-a23621ed6e11.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_709c4704-f768-4640-a198-a23621ed6e11.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 55,\n 48,\n 57,\n 99,\n 52,\n 55,\n 48,\n 52,\n 45,\n 102,\n 55,\n 54,\n 56,\n 45,\n 52,\n 54,\n 52,\n 48,\n 45,\n 97,\n 49,\n 57,\n 56,\n 45,\n 97,\n 50,\n 51,\n 54,\n 50,\n 49,\n 101,\n 100,\n 54,\n 101,\n 49,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_709c4704-f768-4640-a198-a23621ed6e11.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T21:00:00.653762755Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::4]:57992","req_id":"29b59480-0d50-4435-bc54-d965d81f1568","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50316,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_709c4704-f768-4640-a198-a23621ed6e11.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_709c4704-f768-4640-a198-a23621ed6e11.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 55,\n 48,\n 57,\n 99,\n 52,\n 55,\n 48,\n 52,\n 45,\n 102,\n 55,\n 54,\n 56,\n 45,\n 52,\n 54,\n 52,\n 48,\n 45,\n 97,\n 49,\n 57,\n 56,\n 45,\n 97,\n 50,\n 51,\n 54,\n 50,\n 49,\n 101,\n 100,\n 54,\n 101,\n 49,\n 49,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_709c4704-f768-4640-a198-a23621ed6e11.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T21:00:00.654258399Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::4]:57087","req_id":"495bfd64-1ce2-4785-a553-adc380c86c9c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46425,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_ea49db54-0bfa-4e3d-bc93-404c287645e6.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_ea49db54-0bfa-4e3d-bc93-404c287645e6.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 101,\n 97,\n 52,\n 57,\n 100,\n 98,\n 53,\n 52,\n 45,\n 48,\n 98,\n 102,\n 97,\n 45,\n 52,\n 101,\n 51,\n 100,\n 45,\n 98,\n 99,\n 57,\n 51,\n 45,\n 52,\n 48,\n 52,\n 99,\n 50,\n 56,\n 55,\n 54,\n 52,\n 53,\n 101,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_ea49db54-0bfa-4e3d-bc93-404c287645e6.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T21:00:00.676864928Z","hostname":"oxz_internal_dns_7f3bc00f-0320-479a-a8fb-f67e80ed5929","pid":5500,"peer_addr":"[fd00:1122:3344:101::1b]:37790","req_id":"4238c219-72f7-451f-b664-9ac7630ff0a3","component":"dns"}