| 1 | | [ May 28 01:12:26 Enabled. ] |
| 2 | | [ May 28 01:12:26 Rereading configuration. ] |
| 3 | | [ May 28 01:12:26 Rereading configuration. ] |
| 4 | | [ May 28 01:12:28 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:3::1]:5353 --dns-address [fd00:1122:3344:3::1]:53 &"). ] |
| 5 | | [ May 28 01:12:28 Method "start" exited with status 0. ] |
| 6 | | Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } |
| 7 | | note: configured to log to "/dev/stdout" |
| 8 | 2026-05-28T01:12:28.062Z | INFO | dns-server: config
config = Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } }
dns_config = Config { bind_address: [fd00:1122:3344:3::1]:53 }
|
| 9 | 2026-05-28T01:12:28.062Z | INFO | dns-server (store): opening sled database
path = /data/dns
|
| 10 | 2026-05-28T01:12:28.083Z | INFO | dns-server (store): pruning trees for generations newer than 0
|
| 11 | 2026-05-28T01:12:28.083Z | INFO | dns-server (store): pruning trees for generations older than 0
keep = 3
|
| 12 | 2026-05-28T01:12:28.083Z | INFO | dns-server (dns): DNS server bound to address
local_address = [fd00:1122:3344:3::1]:53
|
| 13 | 2026-05-28T01:12:28.083Z | INFO | dns-server (http): listening
local_addr = [fd00:1122:3344:3::1]:5353
|
| 14 | 2026-05-28T01:12:28.976Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:3::1]:5353
remote_addr = [fd00:1122:3344:3::2]:41188
|
| 15 | 2026-05-28T01:12:28.977Z | INFO | dns-server (store): attempting generation update
new_generation = 1
req_id = 9aa17905-8d5c-4cef-8ae2-51cf8f40fc4c
|
| 16 | 2026-05-28T01:12:28.977Z | INFO | dns-server (store): pruning trees for generations newer than 0
|
| 17 | 2026-05-28T01:12:28.990Z | INFO | dns-server (store): pruning trees for generations older than 1
keep = 3
|
| 18 | 2026-05-28T01:12:28.990Z | INFO | dns-server (store): updated generation
new_generation = 1
req_id = 9aa17905-8d5c-4cef-8ae2-51cf8f40fc4c
|
| 19 | 2026-05-28T01:12:28.990Z | INFO | dns-server (http): request completed
latency_us = 13315
local_addr = [fd00:1122:3344:3::1]:5353
method = PUT
remote_addr = [fd00:1122:3344:3::2]:41188
req_id = 9aa17905-8d5c-4cef-8ae2-51cf8f40fc4c
response_code = 204
uri = /config
|
| 20 | 2026-05-28T01:12:40.567Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 40174,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("0.pool.ntp.org."),
),
original: Query {
name: Name("0.pool.ntp.org."),
query_type: AAAA,
query_class: IN,
},
},
],
original: [
1,
48,
4,
112,
111,
111,
108,
3,
110,
116,
112,
3,
111,
114,
103,
0,
0,
28,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "0.pool.ntp.org."
peer_addr = [fd00:1122:3344:101::1b]:41692
req_id = 34f16469-1edd-4ed4-95ff-17b1026ca022
|
| 21 | 2026-05-28T01:12:40.579Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 59626,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("0.pool.ntp.org."),
),
original: Query {
name: Name("0.pool.ntp.org."),
query_type: AAAA,
query_class: IN,
},
},
],
original: [
1,
48,
4,
112,
111,
111,
108,
3,
110,
116,
112,
3,
111,
114,
103,
0,
0,
28,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "0.pool.ntp.org."
peer_addr = [fd00:1122:3344:101::1b]:33992
req_id = 33bd9d1b-6be3-4ab9-856d-19a09f0880e9
|
| 22 | 2026-05-28T01:12:40.595Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 61696,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("0.pool.ntp.org."),
),
original: Query {
name: Name("0.pool.ntp.org."),
query_type: A,
query_class: IN,
},
},
],
original: [
1,
48,
4,
112,
111,
111,
108,
3,
110,
116,
112,
3,
111,
114,
103,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "0.pool.ntp.org."
peer_addr = [fd00:1122:3344:101::1b]:49716
req_id = 55f019b9-f36b-4a13-9ccc-f23f65caef4d
|
| 23 | 2026-05-28T01:13:58.579Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 31763,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:51180
req_id = 0815308e-ed75-404f-a64e-7cf3f837143f
|
| 24 | 2026-05-28T01:14:02.261Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 57029,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
),
original: Query {
name: Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
query_type: A,
query_class: IN,
},
},
],
original: [
51,
111,
120,
122,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
95,
97,
102,
101,
97,
51,
101,
50,
102,
45,
99,
99,
50,
51,
45,
52,
101,
100,
52,
45,
98,
57,
50,
99,
45,
48,
54,
98,
53,
55,
49,
99,
55,
50,
48,
98,
101,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."
peer_addr = [fd00:1122:3344:101::e]:35844
req_id = 5fb9699b-3b49-4cf3-9ff0-07aff6ce2724
|
| 25 | 2026-05-28T01:14:02.261Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 57029,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
),
original: Query {
name: Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
query_type: A,
query_class: IN,
},
},
],
original: [
51,
111,
120,
122,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
95,
97,
102,
101,
97,
51,
101,
50,
102,
45,
99,
99,
50,
51,
45,
52,
101,
100,
52,
45,
98,
57,
50,
99,
45,
48,
54,
98,
53,
55,
49,
99,
55,
50,
48,
98,
101,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."
peer_addr = [fd00:1122:3344:101::e]:47911
req_id = f42d3a4d-4bf3-450e-9145-485eb0355151
|
| 26 | 2026-05-28T01:14:04.685Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 16162,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("0.pool.ntp.org."),
),
original: Query {
name: Name("0.pool.ntp.org."),
query_type: AAAA,
query_class: IN,
},
},
],
original: [
1,
48,
4,
112,
111,
111,
108,
3,
110,
116,
112,
3,
111,
114,
103,
0,
0,
28,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "0.pool.ntp.org."
peer_addr = [fd00:1122:3344:101::1b]:50200
req_id = 2222ae52-550e-40ce-8656-03a7b49b20cb
|
| 27 | 2026-05-28T01:14:04.698Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 19655,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("0.pool.ntp.org."),
),
original: Query {
name: Name("0.pool.ntp.org."),
query_type: AAAA,
query_class: IN,
},
},
],
original: [
1,
48,
4,
112,
111,
111,
108,
3,
110,
116,
112,
3,
111,
114,
103,
0,
0,
28,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "0.pool.ntp.org."
peer_addr = [fd00:1122:3344:101::1b]:42071
req_id = 2e262445-fc3f-4bb0-9fb0-09d8607cfd47
|
| 28 | 2026-05-28T01:14:04.715Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 33218,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("0.pool.ntp.org."),
),
original: Query {
name: Name("0.pool.ntp.org."),
query_type: A,
query_class: IN,
},
},
],
original: [
1,
48,
4,
112,
111,
111,
108,
3,
110,
116,
112,
3,
111,
114,
103,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "0.pool.ntp.org."
peer_addr = [fd00:1122:3344:101::1b]:33125
req_id = f52d6612-fb5b-4427-910a-bc23565c5eeb
|
| 29 | 2026-05-28T01:14:08.521Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 57977,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::a]:31788
req_id = 6bf5fc45-4d40-4dcc-ae29-5bfc401392a9
|
| 30 | 2026-05-28T01:14:08.555Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 24685,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:55076
req_id = 3f69ff26-d46d-47f9-998d-679e5ac4ce42
|
| 31 | 2026-05-28T01:14:09.520Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 13657,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::b]:59108
req_id = f4d1979b-f7e9-4f88-a8ac-79478fc31f5b
|
| 32 | 2026-05-28T01:14:09.794Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 59981,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::c]:10458
req_id = 955ada63-609e-4ed5-805b-9d3ee553b7aa
|
| 33 | 2026-05-28T01:14:18.559Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 3864,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:56802
req_id = 227db819-bbb7-4248-8b34-f8f323946cd2
|
| 34 | 2026-05-28T01:14:28.561Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 43001,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:50022
req_id = 41b22fd6-9332-40df-99f6-e2b1bbd2c97f
|
| 35 | 2026-05-28T01:14:38.563Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 41508,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:64390
req_id = 11ae887b-8051-4652-9dec-415037ae526a
|
| 36 | 2026-05-28T01:14:47.319Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 25054,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::b]:64300
req_id = 0250f5b3-4fd5-413e-894b-ae143e6c795f
|
| 37 | 2026-05-28T01:14:47.557Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:3::1]:5353
remote_addr = [fd00:1122:3344:101::b]:59465
|
| 38 | 2026-05-28T01:14:47.621Z | INFO | dns-server (http): request completed
latency_us = 38636
local_addr = [fd00:1122:3344:3::1]:5353
method = GET
remote_addr = [fd00:1122:3344:101::b]:59465
req_id = 5e33ca4e-677c-41b8-968e-5b98440b0ae0
response_code = 200
uri = /config
|
| 39 | 2026-05-28T01:14:48.565Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 21711,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:54229
req_id = 6201f215-9890-4a1f-84f4-fc1771db9d21
|
| 40 | 2026-05-28T01:14:54.762Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 63332,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::b]:17117
req_id = fa24247e-aa45-48fa-9418-5d6b4ee84c8b
|
| 41 | 2026-05-28T01:14:54.995Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:3::1]:5353
remote_addr = [fd00:1122:3344:101::b]:48663
|
| 42 | 2026-05-28T01:14:54.996Z | INFO | dns-server (http): request completed
latency_us = 269
local_addr = [fd00:1122:3344:3::1]:5353
method = GET
remote_addr = [fd00:1122:3344:101::b]:48663
req_id = d92c8f5a-5ec4-409f-b613-e647602fd797
response_code = 200
uri = /config
|
| 43 | 2026-05-28T01:14:55.005Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:3::1]:5353
remote_addr = [fd00:1122:3344:101::b]:47002
|
| 44 | 2026-05-28T01:14:55.010Z | INFO | dns-server (store): attempting generation update
new_generation = 1
req_id = 13bc615b-c960-4bff-b496-122e55f6a46a
|
| 45 | 2026-05-28T01:14:55.010Z | INFO | dns-server (store): updated generation
new_generation = 1
req_id = 13bc615b-c960-4bff-b496-122e55f6a46a
|
| 46 | 2026-05-28T01:14:55.010Z | INFO | dns-server (http): request completed
latency_us = 3139
local_addr = [fd00:1122:3344:3::1]:5353
method = PUT
remote_addr = [fd00:1122:3344:101::b]:47002
req_id = 13bc615b-c960-4bff-b496-122e55f6a46a
response_code = 204
uri = /config
|
| 47 | 2026-05-28T01:14:55.958Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 10425,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::b]:20598
req_id = 20673435-5be2-4c44-9f99-fe46da7ae607
|
| 48 | 2026-05-28T01:14:56.083Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:3::1]:5353
remote_addr = [fd00:1122:3344:101::b]:38318
|
| 49 | 2026-05-28T01:14:56.083Z | INFO | dns-server (http): request completed
latency_us = 225
local_addr = [fd00:1122:3344:3::1]:5353
method = GET
remote_addr = [fd00:1122:3344:101::b]:38318
req_id = 8c165bbc-668c-4b31-8cca-6f12346c4c52
response_code = 200
uri = /config
|
| 50 | 2026-05-28T01:14:58.566Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 49051,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:59547
req_id = 4ec39a32-84a5-45b4-9df2-3a08e310520c
|
| 51 | 2026-05-28T01:14:59.327Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 37546,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::a]:43937
req_id = ec1e51a9-3b94-4cd3-b306-d042553c855f
|
| 52 | 2026-05-28T01:14:59.534Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:3::1]:5353
remote_addr = [fd00:1122:3344:101::a]:60377
|
| 53 | 2026-05-28T01:14:59.534Z | INFO | dns-server (http): request completed
latency_us = 251
local_addr = [fd00:1122:3344:3::1]:5353
method = GET
remote_addr = [fd00:1122:3344:101::a]:60377
req_id = 0a975fb9-2500-4969-bdd2-cee1b9f6c9a7
response_code = 200
uri = /config
|
| 54 | 2026-05-28T01:15:00.327Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 35472,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4."),
),
original: Query {
name: Name("oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
54,
50,
99,
102,
54,
48,
57,
56,
45,
102,
50,
51,
54,
45,
52,
99,
52,
100,
45,
98,
97,
53,
50,
45,
54,
100,
99,
98,
55,
53,
48,
49,
101,
55,
102,
52,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4."
peer_addr = [fd00:1122:3344:101::6]:54950
req_id = 215db812-6f6e-4e61-8fa6-13b6286ef9f4
|
| 55 | 2026-05-28T01:15:00.327Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 35472,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4."),
),
original: Query {
name: Name("oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
54,
50,
99,
102,
54,
48,
57,
56,
45,
102,
50,
51,
54,
45,
52,
99,
52,
100,
45,
98,
97,
53,
50,
45,
54,
100,
99,
98,
55,
53,
48,
49,
101,
55,
102,
52,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4."
peer_addr = [fd00:1122:3344:101::6]:62145
req_id = b1dedaf0-4853-4a98-a513-f0eefe491ec0
|
| 56 | 2026-05-28T01:15:00.385Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 55261,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_switch."),
),
original: Query {
name: Name("oxz_switch."),
query_type: A,
query_class: IN,
},
},
],
original: [
10,
111,
120,
122,
95,
115,
119,
105,
116,
99,
104,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_switch."
peer_addr = [fd00:1122:3344:101::2]:53141
req_id = 10abf632-407e-4a4a-b3b1-050721d3087a
|
| 57 | 2026-05-28T01:15:00.386Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 55261,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_switch."),
),
original: Query {
name: Name("oxz_switch."),
query_type: A,
query_class: IN,
},
},
],
original: [
10,
111,
120,
122,
95,
115,
119,
105,
116,
99,
104,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_switch."
peer_addr = [fd00:1122:3344:101::2]:61040
req_id = 5a1cc38c-aee5-461c-bb8d-9800834267ac
|
| 58 | 2026-05-28T01:15:00.509Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 28629,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
),
original: Query {
name: Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
query_type: A,
query_class: IN,
},
},
],
original: [
51,
111,
120,
122,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
95,
97,
102,
101,
97,
51,
101,
50,
102,
45,
99,
99,
50,
51,
45,
52,
101,
100,
52,
45,
98,
57,
50,
99,
45,
48,
54,
98,
53,
55,
49,
99,
55,
50,
48,
98,
101,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."
peer_addr = [fd00:1122:3344:101::e]:52964
req_id = b652cd3b-2be2-4419-88d5-e7bd3c22a551
|
| 59 | 2026-05-28T01:15:00.510Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 28629,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
),
original: Query {
name: Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
query_type: A,
query_class: IN,
},
},
],
original: [
51,
111,
120,
122,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
95,
97,
102,
101,
97,
51,
101,
50,
102,
45,
99,
99,
50,
51,
45,
52,
101,
100,
52,
45,
98,
57,
50,
99,
45,
48,
54,
98,
53,
55,
49,
99,
55,
50,
48,
98,
101,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."
peer_addr = [fd00:1122:3344:101::e]:39375
req_id = ba57bdcb-aaa8-4240-84cf-18a55eeb1a62
|
| 60 | 2026-05-28T01:15:00.516Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 32243,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2."),
),
original: Query {
name: Name("oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
55,
99,
48,
101,
97,
49,
52,
97,
45,
97,
48,
48,
49,
45,
52,
102,
48,
101,
45,
97,
57,
50,
52,
45,
49,
54,
100,
52,
50,
51,
51,
101,
100,
98,
101,
50,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2."
peer_addr = [fd00:1122:3344:101::4]:34520
req_id = 8916d2be-9b0a-45ce-8d7f-157de19aacfa
|
| 61 | 2026-05-28T01:15:00.517Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 32243,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2."),
),
original: Query {
name: Name("oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
55,
99,
48,
101,
97,
49,
52,
97,
45,
97,
48,
48,
49,
45,
52,
102,
48,
101,
45,
97,
57,
50,
52,
45,
49,
54,
100,
52,
50,
51,
51,
101,
100,
98,
101,
50,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2."
peer_addr = [fd00:1122:3344:101::4]:46768
req_id = 7b638d22-ebf8-4d9e-877b-57d564d3db48
|
| 62 | 2026-05-28T01:15:00.523Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 52648,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::c]:63775
req_id = 9aa09218-160e-4fa2-b81e-d63a4f68e26f
|
| 63 | 2026-05-28T01:15:00.536Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 47615,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee."),
),
original: Query {
name: Name("oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
48,
99,
98,
52,
101,
56,
101,
102,
45,
54,
56,
97,
53,
45,
52,
56,
100,
98,
45,
56,
55,
54,
100,
45,
99,
102,
49,
97,
98,
101,
49,
101,
50,
56,
101,
101,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee."
peer_addr = [fd00:1122:3344:101::3]:38881
req_id = 30254f26-ea09-45a1-b554-bf6f2fcaff41
|
| 64 | 2026-05-28T01:15:00.537Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 47615,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee."),
),
original: Query {
name: Name("oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
48,
99,
98,
52,
101,
56,
101,
102,
45,
54,
56,
97,
53,
45,
52,
56,
100,
98,
45,
56,
55,
54,
100,
45,
99,
102,
49,
97,
98,
101,
49,
101,
50,
56,
101,
101,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee."
peer_addr = [fd00:1122:3344:101::3]:43916
req_id = a288cc13-b363-41aa-9873-9de810da457a
|
| 65 | 2026-05-28T01:15:00.544Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 4239,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126."),
),
original: Query {
name: Name("oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
57,
55,
48,
51,
97,
97,
48,
53,
45,
53,
51,
54,
98,
45,
52,
51,
49,
48,
45,
56,
100,
52,
56,
45,
55,
57,
52,
98,
100,
57,
102,
102,
56,
49,
50,
54,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126."
peer_addr = [fd00:1122:3344:101::7]:59839
req_id = 4ac571a0-fd6e-465c-b71b-20de575bf73f
|
| 66 | 2026-05-28T01:15:00.545Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 4239,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126."),
),
original: Query {
name: Name("oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
57,
55,
48,
51,
97,
97,
48,
53,
45,
53,
51,
54,
98,
45,
52,
51,
49,
48,
45,
56,
100,
52,
56,
45,
55,
57,
52,
98,
100,
57,
102,
102,
56,
49,
50,
54,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126."
peer_addr = [fd00:1122:3344:101::7]:34829
req_id = 24aa1965-6fdf-4d45-8be1-ac701531ecc5
|
| 67 | 2026-05-28T01:15:00.565Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 62175,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30."),
),
original: Query {
name: Name("oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
51,
49,
57,
49,
98,
55,
52,
48,
45,
55,
100,
99,
55,
45,
52,
52,
51,
100,
45,
56,
99,
97,
48,
45,
100,
51,
55,
102,
99,
49,
99,
101,
101,
99,
51,
48,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30."
peer_addr = [fd00:1122:3344:101::5]:42652
req_id = 11c07dc4-6bfa-44a0-a320-bf2fcdc7e082
|
| 68 | 2026-05-28T01:15:00.566Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 62175,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30."),
),
original: Query {
name: Name("oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
51,
49,
57,
49,
98,
55,
52,
48,
45,
55,
100,
99,
55,
45,
52,
52,
51,
100,
45,
56,
99,
97,
48,
45,
100,
51,
55,
102,
99,
49,
99,
101,
101,
99,
51,
48,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30."
peer_addr = [fd00:1122:3344:101::5]:36987
req_id = 2cd21d3f-27ac-4cf8-adb8-6929a6d63868
|
| 69 | 2026-05-28T01:15:00.651Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:3::1]:5353
remote_addr = [fd00:1122:3344:101::c]:38886
|
| 70 | 2026-05-28T01:15:00.651Z | INFO | dns-server (http): request completed
latency_us = 194
local_addr = [fd00:1122:3344:3::1]:5353
method = GET
remote_addr = [fd00:1122:3344:101::c]:38886
req_id = 3043f7ec-8027-44dc-993b-cf836a71d382
response_code = 200
uri = /config
|
| 71 | 2026-05-28T01:15:00.919Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 39768,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_ntp_232ef14c-e91e-40df-9b38-b5764fde289c."),
),
original: Query {
name: Name("oxz_ntp_232ef14c-e91e-40df-9b38-b5764fde289c."),
query_type: A,
query_class: IN,
},
},
],
original: [
44,
111,
120,
122,
95,
110,
116,
112,
95,
50,
51,
50,
101,
102,
49,
52,
99,
45,
101,
57,
49,
101,
45,
52,
48,
100,
102,
45,
57,
98,
51,
56,
45,
98,
53,
55,
54,
52,
102,
100,
101,
50,
56,
57,
99,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_ntp_232ef14c-e91e-40df-9b38-b5764fde289c."
peer_addr = [fd00:1122:3344:101::1b]:45060
req_id = 03179ba7-4a00-4673-99d5-6639333cb488
|