| 1 | | [ May 28 01:12:26 Enabled. ] |
| 2 | | [ May 28 01:12:26 Rereading configuration. ] |
| 3 | | [ May 28 01:12:26 Rereading configuration. ] |
| 4 | | [ May 28 01:12:27 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:2::1]:5353 --dns-address [fd00:1122:3344:2::1]:53 &"). ] |
| 5 | | [ May 28 01:12:28 Method "start" exited with status 0. ] |
| 6 | | Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } |
| 7 | | note: configured to log to "/dev/stdout" |
| 8 | 2026-05-28T01:12:28.018Z | INFO | dns-server: config
config = Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } }
dns_config = Config { bind_address: [fd00:1122:3344:2::1]:53 }
|
| 9 | 2026-05-28T01:12:28.018Z | INFO | dns-server (store): opening sled database
path = /data/dns
|
| 10 | 2026-05-28T01:12:28.041Z | INFO | dns-server (store): pruning trees for generations newer than 0
|
| 11 | 2026-05-28T01:12:28.041Z | INFO | dns-server (store): pruning trees for generations older than 0
keep = 3
|
| 12 | 2026-05-28T01:12:28.041Z | INFO | dns-server (dns): DNS server bound to address
local_address = [fd00:1122:3344:2::1]:53
|
| 13 | 2026-05-28T01:12:28.041Z | INFO | dns-server (http): listening
local_addr = [fd00:1122:3344:2::1]:5353
|
| 14 | 2026-05-28T01:12:28.042Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 40212,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_nexus._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_nexus._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
6,
95,
110,
101,
120,
117,
115,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal."
peer_addr = [fd00:1122:3344:101::2]:15664
req_id = cab93ffa-4446-4861-a01d-77640eacb3c5
|
| 15 | 2026-05-28T01:12:28.991Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:2::1]:5353
remote_addr = [fd00:1122:3344:2::2]:61645
|
| 16 | 2026-05-28T01:12:28.992Z | INFO | dns-server (store): attempting generation update
new_generation = 1
req_id = 292ea58d-a18e-40cb-b3fb-45b7cfd5ce6f
|
| 17 | 2026-05-28T01:12:28.992Z | INFO | dns-server (store): pruning trees for generations newer than 0
|
| 18 | 2026-05-28T01:12:29.005Z | INFO | dns-server (store): pruning trees for generations older than 1
keep = 3
|
| 19 | 2026-05-28T01:12:29.005Z | INFO | dns-server (store): updated generation
new_generation = 1
req_id = 292ea58d-a18e-40cb-b3fb-45b7cfd5ce6f
|
| 20 | 2026-05-28T01:12:29.005Z | INFO | dns-server (http): request completed
latency_us = 13226
local_addr = [fd00:1122:3344:2::1]:5353
method = PUT
remote_addr = [fd00:1122:3344:2::2]:61645
req_id = 292ea58d-a18e-40cb-b3fb-45b7cfd5ce6f
response_code = 204
uri = /config
|
| 21 | 2026-05-28T01:12:40.567Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 40174,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("0.pool.ntp.org."),
),
original: Query {
name: Name("0.pool.ntp.org."),
query_type: AAAA,
query_class: IN,
},
},
],
original: [
1,
48,
4,
112,
111,
111,
108,
3,
110,
116,
112,
3,
111,
114,
103,
0,
0,
28,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "0.pool.ntp.org."
peer_addr = [fd00:1122:3344:101::1b]:45320
req_id = 64ed1a46-8516-4e56-85dc-91d731234d11
|
| 22 | 2026-05-28T01:12:40.579Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 59626,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("0.pool.ntp.org."),
),
original: Query {
name: Name("0.pool.ntp.org."),
query_type: AAAA,
query_class: IN,
},
},
],
original: [
1,
48,
4,
112,
111,
111,
108,
3,
110,
116,
112,
3,
111,
114,
103,
0,
0,
28,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "0.pool.ntp.org."
peer_addr = [fd00:1122:3344:101::1b]:51270
req_id = abf33ba6-07c1-4815-b555-ff6864d2bfa1
|
| 23 | 2026-05-28T01:12:40.595Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 61696,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("0.pool.ntp.org."),
),
original: Query {
name: Name("0.pool.ntp.org."),
query_type: A,
query_class: IN,
},
},
],
original: [
1,
48,
4,
112,
111,
111,
108,
3,
110,
116,
112,
3,
111,
114,
103,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "0.pool.ntp.org."
peer_addr = [fd00:1122:3344:101::1b]:34384
req_id = 4d2834ec-b8dc-4bb7-a087-65aeaeaa4e6a
|
| 24 | 2026-05-28T01:13:58.579Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 43915,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:63083
req_id = 70327e39-413e-49dd-a9c4-bf2dc293d4e9
|
| 25 | 2026-05-28T01:14:02.255Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 57029,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
),
original: Query {
name: Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
query_type: A,
query_class: IN,
},
},
],
original: [
51,
111,
120,
122,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
95,
97,
102,
101,
97,
51,
101,
50,
102,
45,
99,
99,
50,
51,
45,
52,
101,
100,
52,
45,
98,
57,
50,
99,
45,
48,
54,
98,
53,
55,
49,
99,
55,
50,
48,
98,
101,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."
peer_addr = [fd00:1122:3344:101::e]:63798
req_id = 81e1a7c6-5bce-44d8-86d8-ac2f25455e0e
|
| 26 | 2026-05-28T01:14:02.261Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 57029,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
),
original: Query {
name: Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
query_type: A,
query_class: IN,
},
},
],
original: [
51,
111,
120,
122,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
95,
97,
102,
101,
97,
51,
101,
50,
102,
45,
99,
99,
50,
51,
45,
52,
101,
100,
52,
45,
98,
57,
50,
99,
45,
48,
54,
98,
53,
55,
49,
99,
55,
50,
48,
98,
101,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."
peer_addr = [fd00:1122:3344:101::e]:33874
req_id = f7a08455-0707-4b19-b155-b5ea0da7cfa1
|
| 27 | 2026-05-28T01:14:04.684Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 16162,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("0.pool.ntp.org."),
),
original: Query {
name: Name("0.pool.ntp.org."),
query_type: AAAA,
query_class: IN,
},
},
],
original: [
1,
48,
4,
112,
111,
111,
108,
3,
110,
116,
112,
3,
111,
114,
103,
0,
0,
28,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "0.pool.ntp.org."
peer_addr = [fd00:1122:3344:101::1b]:61572
req_id = a4e4f465-18b2-4c4b-b3c7-4c43b1a9f2ab
|
| 28 | 2026-05-28T01:14:04.698Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 19655,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("0.pool.ntp.org."),
),
original: Query {
name: Name("0.pool.ntp.org."),
query_type: AAAA,
query_class: IN,
},
},
],
original: [
1,
48,
4,
112,
111,
111,
108,
3,
110,
116,
112,
3,
111,
114,
103,
0,
0,
28,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "0.pool.ntp.org."
peer_addr = [fd00:1122:3344:101::1b]:63339
req_id = 8a81c678-3ed7-4e0d-92a1-47dad34a88d4
|
| 29 | 2026-05-28T01:14:04.715Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 33218,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("0.pool.ntp.org."),
),
original: Query {
name: Name("0.pool.ntp.org."),
query_type: A,
query_class: IN,
},
},
],
original: [
1,
48,
4,
112,
111,
111,
108,
3,
110,
116,
112,
3,
111,
114,
103,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "0.pool.ntp.org."
peer_addr = [fd00:1122:3344:101::1b]:44427
req_id = 883ad9ae-593d-4035-b6fe-8fe2bd985421
|
| 30 | 2026-05-28T01:14:08.521Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 51875,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::a]:46148
req_id = cdc1f102-b834-4387-82ed-deab236295dc
|
| 31 | 2026-05-28T01:14:08.555Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 12555,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:56508
req_id = bf184498-9174-4597-afd3-5658ac359497
|
| 32 | 2026-05-28T01:14:09.520Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 18963,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::b]:13081
req_id = 66a41323-ea9e-40bf-8117-d3ee1ce1ac62
|
| 33 | 2026-05-28T01:14:09.794Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 19782,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::c]:39501
req_id = d636a85c-cec6-41d5-bc91-c330626d9cab
|
| 34 | 2026-05-28T01:14:18.559Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 19073,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:59599
req_id = 032f0bb2-fcaf-4092-85c0-4cae9fc7f58b
|
| 35 | 2026-05-28T01:14:28.560Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 54851,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:61665
req_id = febb7c85-bf10-45b2-805e-33d8dad2da5c
|
| 36 | 2026-05-28T01:14:38.563Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 21191,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:58756
req_id = 817a06ce-0917-44b9-a5e5-6048b2f8500c
|
| 37 | 2026-05-28T01:14:47.320Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 12035,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::b]:31578
req_id = 79995bf1-d133-47b8-b32a-a53788966f68
|
| 38 | 2026-05-28T01:14:47.629Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:2::1]:5353
remote_addr = [fd00:1122:3344:101::b]:34622
|
| 39 | 2026-05-28T01:14:47.705Z | INFO | dns-server (http): request completed
latency_us = 48406
local_addr = [fd00:1122:3344:2::1]:5353
method = GET
remote_addr = [fd00:1122:3344:101::b]:34622
req_id = 7de6c706-eef5-4b84-a20e-92f3e741c70c
response_code = 200
uri = /config
|
| 40 | 2026-05-28T01:14:48.565Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 14477,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:59613
req_id = 8a3f2127-ee8a-4ff8-8bc5-85e84ce1fa0d
|
| 41 | 2026-05-28T01:14:54.762Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 22445,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::b]:8863
req_id = a517003b-16c6-41d0-9b78-1594f9a8014d
|
| 42 | 2026-05-28T01:14:54.996Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:2::1]:5353
remote_addr = [fd00:1122:3344:101::b]:42486
|
| 43 | 2026-05-28T01:14:54.996Z | INFO | dns-server (http): request completed
latency_us = 195
local_addr = [fd00:1122:3344:2::1]:5353
method = GET
remote_addr = [fd00:1122:3344:101::b]:42486
req_id = eb3aeb89-c203-44d8-9ba6-1584105a779f
response_code = 200
uri = /config
|
| 44 | 2026-05-28T01:14:55.005Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:2::1]:5353
remote_addr = [fd00:1122:3344:101::b]:54139
|
| 45 | 2026-05-28T01:14:55.009Z | INFO | dns-server (store): attempting generation update
new_generation = 1
req_id = cb512685-476c-400e-8358-1ca1c0f15840
|
| 46 | 2026-05-28T01:14:55.009Z | INFO | dns-server (store): updated generation
new_generation = 1
req_id = cb512685-476c-400e-8358-1ca1c0f15840
|
| 47 | 2026-05-28T01:14:55.009Z | INFO | dns-server (http): request completed
latency_us = 2959
local_addr = [fd00:1122:3344:2::1]:5353
method = PUT
remote_addr = [fd00:1122:3344:101::b]:54139
req_id = cb512685-476c-400e-8358-1ca1c0f15840
response_code = 204
uri = /config
|
| 48 | 2026-05-28T01:14:55.957Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 23564,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::b]:15597
req_id = a61e60c8-eecd-4654-a7f1-4433599b4942
|
| 49 | 2026-05-28T01:14:56.084Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:2::1]:5353
remote_addr = [fd00:1122:3344:101::b]:63720
|
| 50 | 2026-05-28T01:14:56.084Z | INFO | dns-server (http): request completed
latency_us = 186
local_addr = [fd00:1122:3344:2::1]:5353
method = GET
remote_addr = [fd00:1122:3344:101::b]:63720
req_id = acc4b68e-4605-4079-a1d3-9720d24c1499
response_code = 200
uri = /config
|
| 51 | 2026-05-28T01:14:58.566Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 19785,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:63573
req_id = 879ad2d9-8264-46bd-ab04-40fb218d29bd
|
| 52 | 2026-05-28T01:14:59.327Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 47794,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::a]:52223
req_id = e5aec55e-51d1-4ec2-ac19-c6f11fca9eea
|
| 53 | 2026-05-28T01:14:59.535Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:2::1]:5353
remote_addr = [fd00:1122:3344:101::a]:36863
|
| 54 | 2026-05-28T01:14:59.535Z | INFO | dns-server (http): request completed
latency_us = 225
local_addr = [fd00:1122:3344:2::1]:5353
method = GET
remote_addr = [fd00:1122:3344:101::a]:36863
req_id = bb124f4f-7787-4a6b-8460-cc9f0725646b
response_code = 200
uri = /config
|
| 55 | 2026-05-28T01:15:00.326Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 35472,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4."),
),
original: Query {
name: Name("oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
54,
50,
99,
102,
54,
48,
57,
56,
45,
102,
50,
51,
54,
45,
52,
99,
52,
100,
45,
98,
97,
53,
50,
45,
54,
100,
99,
98,
55,
53,
48,
49,
101,
55,
102,
52,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4."
peer_addr = [fd00:1122:3344:101::6]:37171
req_id = dfd1bcb3-56da-4c70-8204-a9f8e536dc0c
|
| 56 | 2026-05-28T01:15:00.327Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 35472,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4."),
),
original: Query {
name: Name("oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
54,
50,
99,
102,
54,
48,
57,
56,
45,
102,
50,
51,
54,
45,
52,
99,
52,
100,
45,
98,
97,
53,
50,
45,
54,
100,
99,
98,
55,
53,
48,
49,
101,
55,
102,
52,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4."
peer_addr = [fd00:1122:3344:101::6]:63935
req_id = a537147c-d7dc-4c60-a023-7dd17c5e021e
|
| 57 | 2026-05-28T01:15:00.385Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 55261,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_switch."),
),
original: Query {
name: Name("oxz_switch."),
query_type: A,
query_class: IN,
},
},
],
original: [
10,
111,
120,
122,
95,
115,
119,
105,
116,
99,
104,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_switch."
peer_addr = [fd00:1122:3344:101::2]:55117
req_id = 5f1fe00c-f2eb-467d-8315-e2fd36a28a2c
|
| 58 | 2026-05-28T01:15:00.386Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 55261,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_switch."),
),
original: Query {
name: Name("oxz_switch."),
query_type: A,
query_class: IN,
},
},
],
original: [
10,
111,
120,
122,
95,
115,
119,
105,
116,
99,
104,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_switch."
peer_addr = [fd00:1122:3344:101::2]:48432
req_id = 72a59464-277a-449f-8ddf-3bf63665a144
|
| 59 | 2026-05-28T01:15:00.509Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 28629,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
),
original: Query {
name: Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
query_type: A,
query_class: IN,
},
},
],
original: [
51,
111,
120,
122,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
95,
97,
102,
101,
97,
51,
101,
50,
102,
45,
99,
99,
50,
51,
45,
52,
101,
100,
52,
45,
98,
57,
50,
99,
45,
48,
54,
98,
53,
55,
49,
99,
55,
50,
48,
98,
101,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."
peer_addr = [fd00:1122:3344:101::e]:63573
req_id = dfe21dc9-1722-42e2-a661-d019e9354ee0
|
| 60 | 2026-05-28T01:15:00.509Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 28629,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
),
original: Query {
name: Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
query_type: A,
query_class: IN,
},
},
],
original: [
51,
111,
120,
122,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
95,
97,
102,
101,
97,
51,
101,
50,
102,
45,
99,
99,
50,
51,
45,
52,
101,
100,
52,
45,
98,
57,
50,
99,
45,
48,
54,
98,
53,
55,
49,
99,
55,
50,
48,
98,
101,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."
peer_addr = [fd00:1122:3344:101::e]:56741
req_id = 2a575fd5-bab9-4a9d-a9f0-910a05f3954a
|
| 61 | 2026-05-28T01:15:00.516Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 32243,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2."),
),
original: Query {
name: Name("oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
55,
99,
48,
101,
97,
49,
52,
97,
45,
97,
48,
48,
49,
45,
52,
102,
48,
101,
45,
97,
57,
50,
52,
45,
49,
54,
100,
52,
50,
51,
51,
101,
100,
98,
101,
50,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2."
peer_addr = [fd00:1122:3344:101::4]:52845
req_id = eefbdc62-2248-44c2-8bf1-006cb52451ad
|
| 62 | 2026-05-28T01:15:00.517Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 32243,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2."),
),
original: Query {
name: Name("oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
55,
99,
48,
101,
97,
49,
52,
97,
45,
97,
48,
48,
49,
45,
52,
102,
48,
101,
45,
97,
57,
50,
52,
45,
49,
54,
100,
52,
50,
51,
51,
101,
100,
98,
101,
50,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2."
peer_addr = [fd00:1122:3344:101::4]:57139
req_id = f21dd05b-6b11-4ea8-83d0-af1cdb4d611c
|
| 63 | 2026-05-28T01:15:00.523Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 1788,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::c]:22884
req_id = 2832c690-1ced-4226-ad24-a883145e107d
|
| 64 | 2026-05-28T01:15:00.536Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 47615,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee."),
),
original: Query {
name: Name("oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
48,
99,
98,
52,
101,
56,
101,
102,
45,
54,
56,
97,
53,
45,
52,
56,
100,
98,
45,
56,
55,
54,
100,
45,
99,
102,
49,
97,
98,
101,
49,
101,
50,
56,
101,
101,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee."
peer_addr = [fd00:1122:3344:101::3]:52878
req_id = 9e0639cc-730d-4e49-ab36-b7955f541790
|
| 65 | 2026-05-28T01:15:00.537Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 47615,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee."),
),
original: Query {
name: Name("oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
48,
99,
98,
52,
101,
56,
101,
102,
45,
54,
56,
97,
53,
45,
52,
56,
100,
98,
45,
56,
55,
54,
100,
45,
99,
102,
49,
97,
98,
101,
49,
101,
50,
56,
101,
101,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee."
peer_addr = [fd00:1122:3344:101::3]:47751
req_id = 1bbf9c06-bd4b-4e8a-8b34-d53fcf2afdfb
|
| 66 | 2026-05-28T01:15:00.544Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 4239,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126."),
),
original: Query {
name: Name("oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
57,
55,
48,
51,
97,
97,
48,
53,
45,
53,
51,
54,
98,
45,
52,
51,
49,
48,
45,
56,
100,
52,
56,
45,
55,
57,
52,
98,
100,
57,
102,
102,
56,
49,
50,
54,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126."
peer_addr = [fd00:1122:3344:101::7]:52876
req_id = 36db5820-b407-495d-ad01-4fae75623819
|
| 67 | 2026-05-28T01:15:00.545Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 4239,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126."),
),
original: Query {
name: Name("oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
57,
55,
48,
51,
97,
97,
48,
53,
45,
53,
51,
54,
98,
45,
52,
51,
49,
48,
45,
56,
100,
52,
56,
45,
55,
57,
52,
98,
100,
57,
102,
102,
56,
49,
50,
54,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126."
peer_addr = [fd00:1122:3344:101::7]:61241
req_id = 76f05d7d-828b-4053-8882-f6563265f4e1
|
| 68 | 2026-05-28T01:15:00.565Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 62175,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30."),
),
original: Query {
name: Name("oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
51,
49,
57,
49,
98,
55,
52,
48,
45,
55,
100,
99,
55,
45,
52,
52,
51,
100,
45,
56,
99,
97,
48,
45,
100,
51,
55,
102,
99,
49,
99,
101,
101,
99,
51,
48,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30."
peer_addr = [fd00:1122:3344:101::5]:52188
req_id = 3a1e97aa-5e88-4f6d-b183-8837dcd83f7e
|
| 69 | 2026-05-28T01:15:00.565Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 62175,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30."),
),
original: Query {
name: Name("oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
51,
49,
57,
49,
98,
55,
52,
48,
45,
55,
100,
99,
55,
45,
52,
52,
51,
100,
45,
56,
99,
97,
48,
45,
100,
51,
55,
102,
99,
49,
99,
101,
101,
99,
51,
48,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30."
peer_addr = [fd00:1122:3344:101::5]:54030
req_id = 11ca84ec-1912-468b-a9d0-16c724e40dc0
|
| 70 | 2026-05-28T01:15:00.652Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:2::1]:5353
remote_addr = [fd00:1122:3344:101::c]:59220
|
| 71 | 2026-05-28T01:15:00.652Z | INFO | dns-server (http): request completed
latency_us = 236
local_addr = [fd00:1122:3344:2::1]:5353
method = GET
remote_addr = [fd00:1122:3344:101::c]:59220
req_id = 2edbe72a-bf6b-4495-ab51-e2bbd50a78d2
response_code = 200
uri = /config
|
| 72 | 2026-05-28T01:15:00.919Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 39768,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_ntp_232ef14c-e91e-40df-9b38-b5764fde289c."),
),
original: Query {
name: Name("oxz_ntp_232ef14c-e91e-40df-9b38-b5764fde289c."),
query_type: A,
query_class: IN,
},
},
],
original: [
44,
111,
120,
122,
95,
110,
116,
112,
95,
50,
51,
50,
101,
102,
49,
52,
99,
45,
101,
57,
49,
101,
45,
52,
48,
100,
102,
45,
57,
98,
51,
56,
45,
98,
53,
55,
54,
52,
102,
100,
101,
50,
56,
57,
99,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_ntp_232ef14c-e91e-40df-9b38-b5764fde289c."
peer_addr = [fd00:1122:3344:101::1b]:64509
req_id = fdac4e91-0148-4c7b-b53f-2935d13e47fb
|