[ Mar 23 20:12:20 Enabled. ] [ Mar 23 20:12:20 Rereading configuration. ] [ Mar 23 20:12:20 Rereading configuration. ] [ Mar 23 20:12:23 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:1::1]:5353 --dns-address [fd00:1122:3344:1::1]:53 &"). ] [ Mar 23 20:12:23 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:1::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:12:23.949856879Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"dns_config":"Config { bind_address: [fd00:1122:3344:1::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:1::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:12:23.950013121Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:12:23.968769642Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:12:23.968780822Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:12:23.968833053Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"component":"dns","local_address":"[fd00:1122:3344:1::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:12:23.969084495Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:12:24.025101519Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:1::2]:63264"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:12:24.026278239Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"new_generation":1,"req_id":"ff2bc192-c800-4d88-b3b9-e0b73caf1840","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:12:24.026288959Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:12:24.093193997Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:12:24.093223448Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"new_generation":1,"req_id":"ff2bc192-c800-4d88-b3b9-e0b73caf1840","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:12:24.093237158Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"uri":"/config","method":"PUT","req_id":"ff2bc192-c800-4d88-b3b9-e0b73caf1840","remote_addr":"[fd00:1122:3344:1::2]:63264","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":67498,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14140,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:12:37.182429638Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::1b]:62660","req_id":"e99b28d2-89e4-4f09-8630-bf8c9054f3de","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48275,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:12:37.188165187Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::1b]:51469","req_id":"39a14a50-9ba8-4ab2-a146-5818fb71d6ad","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 56495,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:12:37.192320793Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::1b]:35486","req_id":"2a7e7af2-a42e-499b-8818-5bfa68bac2a4","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62671,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:14:13.671315431Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::d]:56805","req_id":"c04e1a3a-5ffa-456f-8467-42c61d7e15e9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43332,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_915f754c-f85c-4b65-b293-9f2dc9a61c62.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_915f754c-f85c-4b65-b293-9f2dc9a61c62.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 57,\n 49,\n 53,\n 102,\n 55,\n 53,\n 52,\n 99,\n 45,\n 102,\n 56,\n 53,\n 99,\n 45,\n 52,\n 98,\n 54,\n 53,\n 45,\n 98,\n 50,\n 57,\n 51,\n 45,\n 57,\n 102,\n 50,\n 100,\n 99,\n 57,\n 97,\n 54,\n 49,\n 99,\n 54,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_915f754c-f85c-4b65-b293-9f2dc9a61c62.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:14:15.091419254Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::e]:44126","req_id":"67f8343a-8eab-44af-9666-89eb6377515b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43332,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_915f754c-f85c-4b65-b293-9f2dc9a61c62.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_915f754c-f85c-4b65-b293-9f2dc9a61c62.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 57,\n 49,\n 53,\n 102,\n 55,\n 53,\n 52,\n 99,\n 45,\n 102,\n 56,\n 53,\n 99,\n 45,\n 52,\n 98,\n 54,\n 53,\n 45,\n 98,\n 50,\n 57,\n 51,\n 45,\n 57,\n 102,\n 50,\n 100,\n 99,\n 57,\n 97,\n 54,\n 49,\n 99,\n 54,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_915f754c-f85c-4b65-b293-9f2dc9a61c62.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:14:15.105943039Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::e]:35719","req_id":"e9e506eb-c60e-4791-bb19-47146da9d708","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 58370,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:14:21.182012489Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::a]:43681","req_id":"f9de6208-bbbc-4631-80ac-3be462d625b6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34574,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:14:22.161364193Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::c]:40338","req_id":"b633553f-81ca-47e7-8b5a-b4982d70e86c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 7516,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:14:22.196673478Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::b]:33499","req_id":"ba38881a-a592-4be9-b6d9-f39cec4dabdc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38226,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:14:23.648390855Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::d]:53283","req_id":"160736e5-107b-4969-8918-5672edfdf708","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 25779,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:14:33.650516994Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::d]:62476","req_id":"326e3293-de92-445e-9be8-db34c8f47d6d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 35351,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:14:36.516370452Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::1b]:58106","req_id":"9963826f-efc3-40a0-8a0a-7275eba4c388","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23563,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:14:36.5219449Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::1b]:34841","req_id":"ef5303de-ae89-4f77-8480-d9b308abf15a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 56472,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:14:36.526122276Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::1b]:51897","req_id":"7ee1cbcf-134a-4491-a5bd-db2a19e41004","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 61283,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:14:41.15160224Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::a]:51692","req_id":"05f00a37-ef26-4102-bb90-a4df842240e8","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:14:41.384830236Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:34771"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:14:41.459240319Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"uri":"/config","method":"GET","req_id":"2f366b80-cdfc-4536-a31d-96f57741daa5","remote_addr":"[fd00:1122:3344:101::a]:34771","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":40998,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:14:43.599671106Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:56323"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:14:43.604974832Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"new_generation":1,"req_id":"c922c26c-748b-44b5-b862-14cedf655630","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:14:43.605057393Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"new_generation":1,"req_id":"c922c26c-748b-44b5-b862-14cedf655630","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:14:43.605072653Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"uri":"/config","method":"PUT","req_id":"c922c26c-748b-44b5-b862-14cedf655630","remote_addr":"[fd00:1122:3344:101::a]:56323","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":2876,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 35545,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:14:43.652318781Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::d]:59911","req_id":"eddb6612-41f5-457a-8f1f-56e5ab3df61e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 51077,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:14:43.842923719Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::a]:52103","req_id":"af646fb8-92b7-4922-810c-1387b87dec11","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:14:43.952879619Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:48602"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:14:43.953133331Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"uri":"/config","method":"GET","req_id":"3ae051ba-4eb4-43aa-9dbb-ee7c1d29ec96","remote_addr":"[fd00:1122:3344:101::a]:48602","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":196,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 55157,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:14:44.406999474Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::a]:43201","req_id":"7a92e6be-3598-4d98-93e2-7df593e50d42","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:14:44.526547477Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:37266"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:14:44.526787439Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"uri":"/config","method":"GET","req_id":"d60b0726-5435-4784-95ef-ded4a19fef73","remote_addr":"[fd00:1122:3344:101::a]:37266","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":191,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59937,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:14:52.815380888Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::b]:32879","req_id":"d79be0d7-11c6-480e-9e4f-5a08658563d4","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 39399,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:14:52.870382853Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::c]:55825","req_id":"7e626ad3-3f2a-47e5-9e52-a2928b406d5f","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:14:52.92782613Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:58654"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:14:52.928237833Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"uri":"/config","method":"GET","req_id":"cf78a4c5-2747-4e17-a214-b70e376ac69b","remote_addr":"[fd00:1122:3344:101::b]:58654","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":227,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:14:52.981615935Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:62360"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-23T20:14:52.981892017Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"uri":"/config","method":"GET","req_id":"8d597a95-f437-499d-bb43-a15e84d21a0e","remote_addr":"[fd00:1122:3344:101::c]:62360","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":221,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53944,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:14:53.6545257Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::d]:54323","req_id":"9c414b4a-64e9-430f-b99c-320410e476eb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23512,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_7f5dd3bb-a5b5-43e8-ab34-b5f79f443e40.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_7f5dd3bb-a5b5-43e8-ab34-b5f79f443e40.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 55,\n 102,\n 53,\n 100,\n 100,\n 51,\n 98,\n 98,\n 45,\n 97,\n 53,\n 98,\n 53,\n 45,\n 52,\n 51,\n 101,\n 56,\n 45,\n 97,\n 98,\n 51,\n 52,\n 45,\n 98,\n 53,\n 102,\n 55,\n 57,\n 102,\n 52,\n 52,\n 51,\n 101,\n 52,\n 48,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_7f5dd3bb-a5b5-43e8-ab34-b5f79f443e40.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:15:00.074932597Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::5]:61965","req_id":"9258442b-0038-48bb-a46a-4916151d6589","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 26936,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_88d90769-b275-4a7e-8051-61e328a0636e.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_88d90769-b275-4a7e-8051-61e328a0636e.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 56,\n 100,\n 57,\n 48,\n 55,\n 54,\n 57,\n 45,\n 98,\n 50,\n 55,\n 53,\n 45,\n 52,\n 97,\n 55,\n 101,\n 45,\n 56,\n 48,\n 53,\n 49,\n 45,\n 54,\n 49,\n 101,\n 51,\n 50,\n 56,\n 97,\n 48,\n 54,\n 51,\n 54,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_88d90769-b275-4a7e-8051-61e328a0636e.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:15:00.074977107Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::3]:37939","req_id":"3a822c52-92fe-41cc-ba01-b637dab89367","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23512,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_7f5dd3bb-a5b5-43e8-ab34-b5f79f443e40.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_7f5dd3bb-a5b5-43e8-ab34-b5f79f443e40.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 55,\n 102,\n 53,\n 100,\n 100,\n 51,\n 98,\n 98,\n 45,\n 97,\n 53,\n 98,\n 53,\n 45,\n 52,\n 51,\n 101,\n 56,\n 45,\n 97,\n 98,\n 51,\n 52,\n 45,\n 98,\n 53,\n 102,\n 55,\n 57,\n 102,\n 52,\n 52,\n 51,\n 101,\n 52,\n 48,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_7f5dd3bb-a5b5-43e8-ab34-b5f79f443e40.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:15:00.075574172Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::5]:33848","req_id":"b83b5da9-33a3-465b-bb7a-3beeed5d5074","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 26936,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_88d90769-b275-4a7e-8051-61e328a0636e.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_88d90769-b275-4a7e-8051-61e328a0636e.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 56,\n 100,\n 57,\n 48,\n 55,\n 54,\n 57,\n 45,\n 98,\n 50,\n 55,\n 53,\n 45,\n 52,\n 97,\n 55,\n 101,\n 45,\n 56,\n 48,\n 53,\n 49,\n 45,\n 54,\n 49,\n 101,\n 51,\n 50,\n 56,\n 97,\n 48,\n 54,\n 51,\n 54,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_88d90769-b275-4a7e-8051-61e328a0636e.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:15:00.075605363Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::3]:61314","req_id":"63ee2405-19a6-41db-a952-dd66d2a320c3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 35745,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_b1919362-9527-447b-a072-844e0792047f.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_b1919362-9527-447b-a072-844e0792047f.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 98,\n 49,\n 57,\n 49,\n 57,\n 51,\n 54,\n 50,\n 45,\n 57,\n 53,\n 50,\n 55,\n 45,\n 52,\n 52,\n 55,\n 98,\n 45,\n 97,\n 48,\n 55,\n 50,\n 45,\n 56,\n 52,\n 52,\n 101,\n 48,\n 55,\n 57,\n 50,\n 48,\n 52,\n 55,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_b1919362-9527-447b-a072-844e0792047f.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:15:00.090955625Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::1b]:49419","req_id":"82eceb27-0564-4118-b816-9c7c32102bcf","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16232,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_0aa86c51-dcf9-4eec-9197-c35878881766.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_0aa86c51-dcf9-4eec-9197-c35878881766.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 48,\n 97,\n 97,\n 56,\n 54,\n 99,\n 53,\n 49,\n 45,\n 100,\n 99,\n 102,\n 57,\n 45,\n 52,\n 101,\n 101,\n 99,\n 45,\n 57,\n 49,\n 57,\n 55,\n 45,\n 99,\n 51,\n 53,\n 56,\n 55,\n 56,\n 56,\n 56,\n 49,\n 55,\n 54,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_0aa86c51-dcf9-4eec-9197-c35878881766.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:15:00.14589974Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::7]:35610","req_id":"99f87278-d9ba-4343-9be9-3ebb2908709d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16232,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_0aa86c51-dcf9-4eec-9197-c35878881766.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_0aa86c51-dcf9-4eec-9197-c35878881766.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 48,\n 97,\n 97,\n 56,\n 54,\n 99,\n 53,\n 49,\n 45,\n 100,\n 99,\n 102,\n 57,\n 45,\n 52,\n 101,\n 101,\n 99,\n 45,\n 57,\n 49,\n 57,\n 55,\n 45,\n 99,\n 51,\n 53,\n 56,\n 55,\n 56,\n 56,\n 56,\n 49,\n 55,\n 54,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_0aa86c51-dcf9-4eec-9197-c35878881766.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:15:00.146400615Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::7]:42265","req_id":"57a8fee9-8bf1-4527-a371-156836d6da18","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34304,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:15:00.404688817Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::2]:48548","req_id":"dfd53203-39a2-456d-b03f-f21fb60a2f5f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34304,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:15:00.405311093Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::2]:33493","req_id":"88af96b6-525e-4086-827b-fa914e3a10f3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48866,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_915f754c-f85c-4b65-b293-9f2dc9a61c62.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_915f754c-f85c-4b65-b293-9f2dc9a61c62.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 57,\n 49,\n 53,\n 102,\n 55,\n 53,\n 52,\n 99,\n 45,\n 102,\n 56,\n 53,\n 99,\n 45,\n 52,\n 98,\n 54,\n 53,\n 45,\n 98,\n 50,\n 57,\n 51,\n 45,\n 57,\n 102,\n 50,\n 100,\n 99,\n 57,\n 97,\n 54,\n 49,\n 99,\n 54,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_915f754c-f85c-4b65-b293-9f2dc9a61c62.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:15:00.599639221Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::e]:35923","req_id":"d8fb79bb-09bb-435b-ab51-94641349efb6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48866,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_915f754c-f85c-4b65-b293-9f2dc9a61c62.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_915f754c-f85c-4b65-b293-9f2dc9a61c62.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 57,\n 49,\n 53,\n 102,\n 55,\n 53,\n 52,\n 99,\n 45,\n 102,\n 56,\n 53,\n 99,\n 45,\n 52,\n 98,\n 54,\n 53,\n 45,\n 98,\n 50,\n 57,\n 51,\n 45,\n 57,\n 102,\n 50,\n 100,\n 99,\n 57,\n 97,\n 54,\n 49,\n 99,\n 54,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_915f754c-f85c-4b65-b293-9f2dc9a61c62.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:15:00.600298157Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::e]:55068","req_id":"722d56c9-99ac-4fd5-9617-a02415cea997","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48482,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_647309ea-4c01-45e0-b676-f1d890640394.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_647309ea-4c01-45e0-b676-f1d890640394.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 52,\n 55,\n 51,\n 48,\n 57,\n 101,\n 97,\n 45,\n 52,\n 99,\n 48,\n 49,\n 45,\n 52,\n 53,\n 101,\n 48,\n 45,\n 98,\n 54,\n 55,\n 54,\n 45,\n 102,\n 49,\n 100,\n 56,\n 57,\n 48,\n 54,\n 52,\n 48,\n 51,\n 57,\n 52,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_647309ea-4c01-45e0-b676-f1d890640394.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:15:00.997216857Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::4]:59615","req_id":"47bd22cb-4c61-4c60-966c-918f266e0357","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48482,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_647309ea-4c01-45e0-b676-f1d890640394.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_647309ea-4c01-45e0-b676-f1d890640394.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 52,\n 55,\n 51,\n 48,\n 57,\n 101,\n 97,\n 45,\n 52,\n 99,\n 48,\n 49,\n 45,\n 52,\n 53,\n 101,\n 48,\n 45,\n 98,\n 54,\n 55,\n 54,\n 45,\n 102,\n 49,\n 100,\n 56,\n 57,\n 48,\n 54,\n 52,\n 48,\n 51,\n 57,\n 52,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_647309ea-4c01-45e0-b676-f1d890640394.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:15:00.997863053Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::4]:57971","req_id":"047268a0-49ba-4f71-b2ee-88f21afc4f0e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 65022,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_7ba0883b-5a00-447c-b1fc-85a4a4985955.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_7ba0883b-5a00-447c-b1fc-85a4a4985955.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 55,\n 98,\n 97,\n 48,\n 56,\n 56,\n 51,\n 98,\n 45,\n 53,\n 97,\n 48,\n 48,\n 45,\n 52,\n 52,\n 55,\n 99,\n 45,\n 98,\n 49,\n 102,\n 99,\n 45,\n 56,\n 53,\n 97,\n 52,\n 97,\n 52,\n 57,\n 56,\n 53,\n 57,\n 53,\n 53,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_7ba0883b-5a00-447c-b1fc-85a4a4985955.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:15:01.029335345Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::6]:52597","req_id":"92d6dcb6-a9ed-4872-8682-845ba9072717","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 65022,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_7ba0883b-5a00-447c-b1fc-85a4a4985955.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_7ba0883b-5a00-447c-b1fc-85a4a4985955.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 55,\n 98,\n 97,\n 48,\n 56,\n 56,\n 51,\n 98,\n 45,\n 53,\n 97,\n 48,\n 48,\n 45,\n 52,\n 52,\n 55,\n 99,\n 45,\n 98,\n 49,\n 102,\n 99,\n 45,\n 56,\n 53,\n 97,\n 52,\n 97,\n 52,\n 57,\n 56,\n 53,\n 57,\n 53,\n 53,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_7ba0883b-5a00-447c-b1fc-85a4a4985955.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-23T20:15:01.02995546Z","hostname":"oxz_internal_dns_8bdc9c08-2e3e-42f1-afbe-18a0afbeebc2","pid":5578,"peer_addr":"[fd00:1122:3344:101::6]:52975","req_id":"e153d7e8-7689-4976-901b-37b90ef253fa","component":"dns"}