[ May 19 02:41:06 Enabled. ] [ May 19 02:41:06 Rereading configuration. ] [ May 19 02:41:07 Rereading configuration. ] [ May 19 02:41:08 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:3::1]:5353 --dns-address [fd00:1122:3344:3::1]:53 &"). ] [ May 19 02:41:08 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:41:08.254940694Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"dns_config":"Config { bind_address: [fd00:1122:3344:3::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:41:08.255079146Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:41:08.275026571Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:41:08.275050791Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:41:08.275120552Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"component":"dns","local_address":"[fd00:1122:3344:3::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:41:08.275530066Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50967,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:41:08.351120041Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::2]:18179","req_id":"b6e53553-7326-42c9-903e-7b303fb5cd69","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:41:10.017143416Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:3::2]:61913"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:41:10.017953264Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"new_generation":1,"req_id":"326ff7cb-ed5b-465a-9043-fffbfc4dc482","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:41:10.018030625Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:41:10.029537074Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:41:10.029573294Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"new_generation":1,"req_id":"326ff7cb-ed5b-465a-9043-fffbfc4dc482","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:41:10.029583184Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"PUT","req_id":"326ff7cb-ed5b-465a-9043-fffbfc4dc482","remote_addr":"[fd00:1122:3344:3::2]:61913","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":12004,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16074,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:41:21.80663808Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::1b]:37964","req_id":"4d72e2ba-1eb5-4674-a426-8d1b5af430e3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 25922,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:41:21.811068806Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::1b]:60806","req_id":"b640f232-dcbf-4116-9848-c3f87a21799a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 15131,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:41:21.815897555Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::1b]:42128","req_id":"58606cf0-0d78-4bdc-a453-0f035a4df6ba","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 33461,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:42:19.200234172Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::d]:52975","req_id":"7178eae0-aa5e-410a-88ac-a3b7104ff343","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 22468,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_e9a92c02-f08f-43a1-ad00-f7f0a7f52852.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_e9a92c02-f08f-43a1-ad00-f7f0a7f52852.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 101,\n 57,\n 97,\n 57,\n 50,\n 99,\n 48,\n 50,\n 45,\n 102,\n 48,\n 56,\n 102,\n 45,\n 52,\n 51,\n 97,\n 49,\n 45,\n 97,\n 100,\n 48,\n 48,\n 45,\n 102,\n 55,\n 102,\n 48,\n 97,\n 55,\n 102,\n 53,\n 50,\n 56,\n 53,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_e9a92c02-f08f-43a1-ad00-f7f0a7f52852.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:42:24.21515967Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::e]:58196","req_id":"25b26264-2de5-45c2-b6ca-a8a05a33a85a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 22468,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_e9a92c02-f08f-43a1-ad00-f7f0a7f52852.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_e9a92c02-f08f-43a1-ad00-f7f0a7f52852.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 101,\n 57,\n 97,\n 57,\n 50,\n 99,\n 48,\n 50,\n 45,\n 102,\n 48,\n 56,\n 102,\n 45,\n 52,\n 51,\n 97,\n 49,\n 45,\n 97,\n 100,\n 48,\n 48,\n 45,\n 102,\n 55,\n 102,\n 48,\n 97,\n 55,\n 102,\n 53,\n 50,\n 56,\n 53,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_e9a92c02-f08f-43a1-ad00-f7f0a7f52852.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:42:24.215664305Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::e]:44613","req_id":"f84ef6eb-860d-4144-8ed9-1e06b36567c1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 33210,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:42:29.178572541Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::d]:54720","req_id":"64d52d0f-3594-4956-b571-8c3075ac8838","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 13365,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:42:29.848519675Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::a]:6187","req_id":"930fdbd8-a9fe-4959-b7b2-f6a0d922b7d7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20713,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:42:29.851603377Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::b]:23717","req_id":"04a786ee-2c0d-4b86-8a2c-6835c63444cc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 63607,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:42:30.34992319Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::c]:48403","req_id":"dd42de90-5313-4743-8c68-52765b18e371","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:42:31.219808556Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:42427"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:42:31.277897502Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"new_generation":1,"req_id":"2a5d0666-160f-4233-bce0-e2ca98badf8c","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:42:31.278015363Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"new_generation":1,"req_id":"2a5d0666-160f-4233-bce0-e2ca98badf8c","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:42:31.278034523Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"PUT","req_id":"2a5d0666-160f-4233-bce0-e2ca98badf8c","remote_addr":"[fd00:1122:3344:101::b]:42427","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":25994,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:42:31.834736156Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:41690"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:42:31.834862957Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:56737"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:42:31.834896628Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:54916"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:42:31.851286136Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"GET","req_id":"ee8ad4bc-98df-4aee-98f4-1db354e4deee","remote_addr":"[fd00:1122:3344:101::a]:56737","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":16341,"response_code":200} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:42:31.851357277Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"GET","req_id":"971590ad-badd-4d21-90e2-01a731b6b1c6","remote_addr":"[fd00:1122:3344:101::b]:41690","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":16161,"response_code":200} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:42:31.851393567Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"GET","req_id":"5e856a5d-0e17-4387-a3fb-c960c508b0d4","remote_addr":"[fd00:1122:3344:101::c]:54916","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":16265,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48990,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:42:32.183540784Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::b]:55674","req_id":"eb71a525-b7b6-46e3-af1f-542d068f636c","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:42:32.308700299Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:47036"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:42:32.308974332Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"GET","req_id":"e8a656d9-382b-43db-8e10-1dd8f2c2351c","remote_addr":"[fd00:1122:3344:101::b]:47036","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":194,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 56230,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:42:32.710022057Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::b]:55261","req_id":"b48c4b49-141c-443d-961d-81f095a9f303","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:42:32.851236206Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:37257"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:42:32.851577269Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"GET","req_id":"93ca40b2-0063-4db1-ab81-54b36a541422","remote_addr":"[fd00:1122:3344:101::b]:37257","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":248,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 11693,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:42:39.180544902Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::d]:57555","req_id":"3955f97f-954b-42b3-9a5e-158ddda331c3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 37242,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:42:40.329441091Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::a]:31022","req_id":"726901de-5eee-4e47-9cb0-da2c1c43f57e","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:42:40.431819841Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:43929"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:42:40.432109634Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"GET","req_id":"fefc1556-14fa-4983-b6cb-5530c38f1aca","remote_addr":"[fd00:1122:3344:101::a]:43929","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":204,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 37704,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:42:40.796579784Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::c]:53972","req_id":"78628264-9418-4139-9a14-422a45afa6d0","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:42:40.896747932Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:49563"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:42:40.897096416Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"GET","req_id":"cfc894fd-140e-4aa9-aea2-d2c678bf7b9a","remote_addr":"[fd00:1122:3344:101::c]:49563","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":227,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 22776,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:42:45.83270201Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::1b]:55479","req_id":"8c29f28a-0734-40f4-aa2f-8fb3fab1d154","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21052,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:42:45.836960524Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::1b]:42132","req_id":"46c006fe-47af-49dd-969c-eeda07132102","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41975,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:42:45.842029106Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::1b]:58161","req_id":"fd9ed450-a408-4ab6-9be9-925b76ba7ae2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 12114,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:42:49.181832676Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::d]:60485","req_id":"60a421e1-2ff7-42d6-a2b0-c0ff8043ed00","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 61661,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:42:59.184648749Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::d]:61494","req_id":"2542113d-0f4d-478e-ad7c-fc7605368cea","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48944,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:43:09.187644119Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::d]:57896","req_id":"c4e591c5-588b-49a1-bffb-eb0a83bcbf70","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3229,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:43:19.189994142Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::d]:62221","req_id":"816d3051-15e0-4006-9ef1-519ed15778fa","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 11136,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:43:29.191386779Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::d]:57150","req_id":"0e1a33af-538a-444c-bc10-e156304872c1","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:43:29.521671508Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:33857"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:43:29.524834341Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"new_generation":1,"req_id":"d0ee981b-0f0a-4a44-afb5-4b478a3107db","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:43:29.524886991Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"new_generation":1,"req_id":"d0ee981b-0f0a-4a44-afb5-4b478a3107db","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:43:29.524900211Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"PUT","req_id":"d0ee981b-0f0a-4a44-afb5-4b478a3107db","remote_addr":"[fd00:1122:3344:101::b]:33857","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":335,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:43:29.669157952Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:33025"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:43:29.672018001Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"new_generation":1,"req_id":"3cf4ccae-a50f-4390-8cd9-48feaa60c78b","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:43:29.672055392Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"new_generation":1,"req_id":"3cf4ccae-a50f-4390-8cd9-48feaa60c78b","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:43:29.672089732Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"PUT","req_id":"3cf4ccae-a50f-4390-8cd9-48feaa60c78b","remote_addr":"[fd00:1122:3344:101::a]:33025","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":400,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:43:30.341333929Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:48643"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:43:30.345870426Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"new_generation":1,"req_id":"00e5a9e7-b901-4c20-adfb-dbc0e8ef54a3","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:43:30.345913526Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"new_generation":1,"req_id":"00e5a9e7-b901-4c20-adfb-dbc0e8ef54a3","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:43:30.345924356Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"PUT","req_id":"00e5a9e7-b901-4c20-adfb-dbc0e8ef54a3","remote_addr":"[fd00:1122:3344:101::c]:48643","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":209,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 65261,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:43:30.553751559Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::a]:34221","req_id":"b7543f49-73bc-467a-9ba0-b13d57bda81e","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:43:30.658565164Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:60408"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:43:30.658785577Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"GET","req_id":"5c9f69ea-74e2-49ab-98f3-ca19170c0353","remote_addr":"[fd00:1122:3344:101::a]:60408","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":184,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54517,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:43:30.98072605Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::c]:30516","req_id":"1882fde7-d77e-463c-b61b-b667d5ac362e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 12074,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:43:30.981250386Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::b]:40956","req_id":"32db3eb4-e25a-4673-aca5-f8a308093965","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:43:31.086822749Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:48313"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:43:31.087087122Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"GET","req_id":"a15e5f36-8638-4d10-a61a-7e6e4b78247e","remote_addr":"[fd00:1122:3344:101::c]:48313","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":212,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:43:31.125818599Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:47116"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:43:31.126026701Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"GET","req_id":"a9d13b5e-bbd4-434c-a080-317b9573b348","remote_addr":"[fd00:1122:3344:101::b]:47116","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":181,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52243,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:43:39.192908898Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::d]:56350","req_id":"84bc3375-e183-44bd-ac1f-c2bd66f1f276","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 49448,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:43:49.194290743Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::d]:64109","req_id":"5a51cac8-2f3d-416c-9dcd-2ac889c34662","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57243,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:43:59.196509419Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::d]:50896","req_id":"a14fca00-65a2-4d41-8e0f-dcd337360e87","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14185,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:44:09.198745813Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::d]:50024","req_id":"e7e089a5-950a-47b6-83b0-8bc884307741","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40669,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:44:19.201006959Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::d]:59500","req_id":"1149825d-68e6-40ae-bf4a-67465768619a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 44953,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:44:29.202381375Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::d]:52460","req_id":"fc336a63-a406-4dd5-bcb3-fb401d077603","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:44:29.507567707Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:35085"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:44:29.510332326Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"new_generation":1,"req_id":"08d5ce39-52a0-4ca9-b6f1-5e2d16b290f8","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:44:29.510362646Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"new_generation":1,"req_id":"08d5ce39-52a0-4ca9-b6f1-5e2d16b290f8","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:44:29.510545008Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"PUT","req_id":"08d5ce39-52a0-4ca9-b6f1-5e2d16b290f8","remote_addr":"[fd00:1122:3344:101::a]:35085","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":236,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:44:29.522708693Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:53783"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:44:29.526146478Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"new_generation":1,"req_id":"48640b7e-05dc-4a56-8dfb-28eedf5ffa0d","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:44:29.526218849Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"new_generation":1,"req_id":"48640b7e-05dc-4a56-8dfb-28eedf5ffa0d","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:44:29.526235129Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"PUT","req_id":"48640b7e-05dc-4a56-8dfb-28eedf5ffa0d","remote_addr":"[fd00:1122:3344:101::b]:53783","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":239,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:44:30.102084158Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:40313"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:44:30.105357771Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"new_generation":1,"req_id":"b593f577-b48e-400b-877a-d4e69f845ad1","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:44:30.105402661Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"new_generation":1,"req_id":"b593f577-b48e-400b-877a-d4e69f845ad1","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:44:30.105545643Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"PUT","req_id":"b593f577-b48e-400b-877a-d4e69f845ad1","remote_addr":"[fd00:1122:3344:101::c]:40313","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":206,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53549,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:44:30.421106451Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::a]:16216","req_id":"bcb9e4ea-8d69-4a1a-9a4b-94753f461bfb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 63882,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:44:30.470392705Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::b]:56785","req_id":"3d63dc9b-f467-4535-a747-10c8d62e4b84","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:44:30.533755737Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:50193"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:44:30.53399254Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"GET","req_id":"ce364501-bbe0-4cca-99b8-885b48824277","remote_addr":"[fd00:1122:3344:101::a]:50193","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":192,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:44:30.568634735Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:33530"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:44:30.568883628Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"GET","req_id":"e325952d-e276-4689-bc63-918663909014","remote_addr":"[fd00:1122:3344:101::b]:33530","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":214,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 60642,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:44:30.966829481Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::c]:59091","req_id":"57948a81-cbbb-4b1d-a37c-3afba029eadd","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:44:31.070462635Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:50121"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-19T02:44:31.070686277Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"uri":"/config","method":"GET","req_id":"e46eba5a-b4e9-434d-8242-6eb5e020e01a","remote_addr":"[fd00:1122:3344:101::c]:50121","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":188,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54800,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:44:39.203937095Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::d]:59375","req_id":"550308be-8ace-408d-9491-b26564922d5f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 56335,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:44:49.20585735Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::d]:58748","req_id":"bcc295cc-146b-4be3-8abe-849ee8109fdd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50910,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:44:59.207621518Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::d]:51921","req_id":"a542efbf-e453-48c6-a4f1-ce1a5cfd3a7a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 49449,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_125ffa43-a002-48f3-a56a-81451730163a.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_125ffa43-a002-48f3-a56a-81451730163a.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 49,\n 50,\n 53,\n 102,\n 102,\n 97,\n 52,\n 51,\n 45,\n 97,\n 48,\n 48,\n 50,\n 45,\n 52,\n 56,\n 102,\n 51,\n 45,\n 97,\n 53,\n 54,\n 97,\n 45,\n 56,\n 49,\n 52,\n 53,\n 49,\n 55,\n 51,\n 48,\n 49,\n 54,\n 51,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_125ffa43-a002-48f3-a56a-81451730163a.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:45:00.148459822Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::1b]:41087","req_id":"f43cd37a-ec83-495b-9bc5-cd1d1561297f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 29257,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_acb4ce84-1a57-4753-9080-ae538f4f8e96.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_acb4ce84-1a57-4753-9080-ae538f4f8e96.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 97,\n 99,\n 98,\n 52,\n 99,\n 101,\n 56,\n 52,\n 45,\n 49,\n 97,\n 53,\n 55,\n 45,\n 52,\n 55,\n 53,\n 51,\n 45,\n 57,\n 48,\n 56,\n 48,\n 45,\n 97,\n 101,\n 53,\n 51,\n 56,\n 102,\n 52,\n 102,\n 56,\n 101,\n 57,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_acb4ce84-1a57-4753-9080-ae538f4f8e96.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:45:00.171240175Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::7]:58160","req_id":"05a8c5db-146a-4221-bd8b-cc157f501645","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 29257,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_acb4ce84-1a57-4753-9080-ae538f4f8e96.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_acb4ce84-1a57-4753-9080-ae538f4f8e96.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 97,\n 99,\n 98,\n 52,\n 99,\n 101,\n 56,\n 52,\n 45,\n 49,\n 97,\n 53,\n 55,\n 45,\n 52,\n 55,\n 53,\n 51,\n 45,\n 57,\n 48,\n 56,\n 48,\n 45,\n 97,\n 101,\n 53,\n 51,\n 56,\n 102,\n 52,\n 102,\n 56,\n 101,\n 57,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_acb4ce84-1a57-4753-9080-ae538f4f8e96.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:45:00.171896172Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::7]:60907","req_id":"f7d941e5-75a8-402a-862f-b24b9f9ccb30","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48821,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_7dcb20ef-0465-42f5-b64d-9146e53782bb.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_7dcb20ef-0465-42f5-b64d-9146e53782bb.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 55,\n 100,\n 99,\n 98,\n 50,\n 48,\n 101,\n 102,\n 45,\n 48,\n 52,\n 54,\n 53,\n 45,\n 52,\n 50,\n 102,\n 53,\n 45,\n 98,\n 54,\n 52,\n 100,\n 45,\n 57,\n 49,\n 52,\n 54,\n 101,\n 53,\n 51,\n 55,\n 56,\n 50,\n 98,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_7dcb20ef-0465-42f5-b64d-9146e53782bb.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:45:00.175318097Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::3]:35977","req_id":"438cac54-a6b0-4542-b20c-374398041bf2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48821,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_7dcb20ef-0465-42f5-b64d-9146e53782bb.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_7dcb20ef-0465-42f5-b64d-9146e53782bb.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 55,\n 100,\n 99,\n 98,\n 50,\n 48,\n 101,\n 102,\n 45,\n 48,\n 52,\n 54,\n 53,\n 45,\n 52,\n 50,\n 102,\n 53,\n 45,\n 98,\n 54,\n 52,\n 100,\n 45,\n 57,\n 49,\n 52,\n 54,\n 101,\n 53,\n 51,\n 55,\n 56,\n 50,\n 98,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_7dcb20ef-0465-42f5-b64d-9146e53782bb.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:45:00.175934324Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::3]:35473","req_id":"75ab9159-8af2-4838-894a-806ef0f2fb1e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 5092,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_b5801a56-8754-41b4-a9bc-5982bd0cb9bc.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_b5801a56-8754-41b4-a9bc-5982bd0cb9bc.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 53,\n 56,\n 48,\n 49,\n 97,\n 53,\n 54,\n 45,\n 56,\n 55,\n 53,\n 52,\n 45,\n 52,\n 49,\n 98,\n 52,\n 45,\n 97,\n 57,\n 98,\n 99,\n 45,\n 53,\n 57,\n 56,\n 50,\n 98,\n 100,\n 48,\n 99,\n 98,\n 57,\n 98,\n 99,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_b5801a56-8754-41b4-a9bc-5982bd0cb9bc.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:45:00.263174319Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::4]:57892","req_id":"2e579241-1cb7-4f59-ab26-2d3f91c8a00a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 5092,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_b5801a56-8754-41b4-a9bc-5982bd0cb9bc.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_b5801a56-8754-41b4-a9bc-5982bd0cb9bc.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 53,\n 56,\n 48,\n 49,\n 97,\n 53,\n 54,\n 45,\n 56,\n 55,\n 53,\n 52,\n 45,\n 52,\n 49,\n 98,\n 52,\n 45,\n 97,\n 57,\n 98,\n 99,\n 45,\n 53,\n 57,\n 56,\n 50,\n 98,\n 100,\n 48,\n 99,\n 98,\n 57,\n 98,\n 99,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_b5801a56-8754-41b4-a9bc-5982bd0cb9bc.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:45:00.263736375Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::4]:55656","req_id":"fc8d89f9-cda5-44da-937a-e86845925016","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 39507,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_e9a92c02-f08f-43a1-ad00-f7f0a7f52852.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_e9a92c02-f08f-43a1-ad00-f7f0a7f52852.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 101,\n 57,\n 97,\n 57,\n 50,\n 99,\n 48,\n 50,\n 45,\n 102,\n 48,\n 56,\n 102,\n 45,\n 52,\n 51,\n 97,\n 49,\n 45,\n 97,\n 100,\n 48,\n 48,\n 45,\n 102,\n 55,\n 102,\n 48,\n 97,\n 55,\n 102,\n 53,\n 50,\n 56,\n 53,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_e9a92c02-f08f-43a1-ad00-f7f0a7f52852.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:45:00.618765738Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::e]:38946","req_id":"ae2563ca-967d-4cb2-be59-fefe56bbb1b3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 39507,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_e9a92c02-f08f-43a1-ad00-f7f0a7f52852.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_e9a92c02-f08f-43a1-ad00-f7f0a7f52852.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 101,\n 57,\n 97,\n 57,\n 50,\n 99,\n 48,\n 50,\n 45,\n 102,\n 48,\n 56,\n 102,\n 45,\n 52,\n 51,\n 97,\n 49,\n 45,\n 97,\n 100,\n 48,\n 48,\n 45,\n 102,\n 55,\n 102,\n 48,\n 97,\n 55,\n 102,\n 53,\n 50,\n 56,\n 53,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_e9a92c02-f08f-43a1-ad00-f7f0a7f52852.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:45:00.619320924Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::e]:52206","req_id":"359eb2c0-d6c9-4be3-a418-90aec375373f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 487,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_ace3d2b6-4e58-44fe-8fe1-a5fb87d9fcb8.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_ace3d2b6-4e58-44fe-8fe1-a5fb87d9fcb8.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 97,\n 99,\n 101,\n 51,\n 100,\n 50,\n 98,\n 54,\n 45,\n 52,\n 101,\n 53,\n 56,\n 45,\n 52,\n 52,\n 102,\n 101,\n 45,\n 56,\n 102,\n 101,\n 49,\n 45,\n 97,\n 53,\n 102,\n 98,\n 56,\n 55,\n 100,\n 57,\n 102,\n 99,\n 98,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_ace3d2b6-4e58-44fe-8fe1-a5fb87d9fcb8.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:45:00.919282892Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::6]:49862","req_id":"1c118fa3-1429-4f5e-a813-b0d9d2e38bab","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 487,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_ace3d2b6-4e58-44fe-8fe1-a5fb87d9fcb8.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_ace3d2b6-4e58-44fe-8fe1-a5fb87d9fcb8.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 97,\n 99,\n 101,\n 51,\n 100,\n 50,\n 98,\n 54,\n 45,\n 52,\n 101,\n 53,\n 56,\n 45,\n 52,\n 52,\n 102,\n 101,\n 45,\n 56,\n 102,\n 101,\n 49,\n 45,\n 97,\n 53,\n 102,\n 98,\n 56,\n 55,\n 100,\n 57,\n 102,\n 99,\n 98,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_ace3d2b6-4e58-44fe-8fe1-a5fb87d9fcb8.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-19T02:45:00.919925069Z","hostname":"oxz_internal_dns_27b8e10a-dbdc-425b-be8a-1fac20b4fe18","pid":5435,"peer_addr":"[fd00:1122:3344:101::6]:34927","req_id":"3182c736-c64b-434d-932d-2ff1ff3ed421","component":"dns"}