| 1 | | [ May 28 01:12:26 Enabled. ] |
| 2 | | [ May 28 01:12:26 Rereading configuration. ] |
| 3 | | [ May 28 01:12:26 Rereading configuration. ] |
| 4 | | [ May 28 01:12:27 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:1::1]:5353 --dns-address [fd00:1122:3344:1::1]:53 &"). ] |
| 5 | | [ May 28 01:12:27 Method "start" exited with status 0. ] |
| 6 | | Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:1::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } |
| 7 | | note: configured to log to "/dev/stdout" |
| 8 | 2026-05-28T01:12:27.998Z | INFO | dns-server: config
config = Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:1::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } }
dns_config = Config { bind_address: [fd00:1122:3344:1::1]:53 }
|
| 9 | 2026-05-28T01:12:27.998Z | INFO | dns-server (store): opening sled database
path = /data/dns
|
| 10 | 2026-05-28T01:12:28.022Z | INFO | dns-server (store): pruning trees for generations newer than 0
|
| 11 | 2026-05-28T01:12:28.022Z | INFO | dns-server (store): pruning trees for generations older than 0
keep = 3
|
| 12 | 2026-05-28T01:12:28.022Z | INFO | dns-server (dns): DNS server bound to address
local_address = [fd00:1122:3344:1::1]:53
|
| 13 | 2026-05-28T01:12:28.023Z | INFO | dns-server (http): listening
local_addr = [fd00:1122:3344:1::1]:5353
|
| 14 | 2026-05-28T01:12:28.042Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 36591,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_nexus._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_nexus._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
6,
95,
110,
101,
120,
117,
115,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} SERVFAIL: server is not authoritative for name: "_nexus._tcp.control-plane.oxide.internal."
peer_addr = [fd00:1122:3344:101::2]:24710
req_id = 1e8a452d-68df-4639-9428-7870e4010415
|
| 15 | 2026-05-28T01:12:29.006Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:1::1]:5353
remote_addr = [fd00:1122:3344:1::2]:42550
|
| 16 | 2026-05-28T01:12:29.007Z | INFO | dns-server (store): attempting generation update
new_generation = 1
req_id = 00235b80-0243-486b-84f4-955f2f993d2e
|
| 17 | 2026-05-28T01:12:29.007Z | INFO | dns-server (store): pruning trees for generations newer than 0
|
| 18 | 2026-05-28T01:12:29.019Z | INFO | dns-server (store): pruning trees for generations older than 1
keep = 3
|
| 19 | 2026-05-28T01:12:29.019Z | INFO | dns-server (store): updated generation
new_generation = 1
req_id = 00235b80-0243-486b-84f4-955f2f993d2e
|
| 20 | 2026-05-28T01:12:29.019Z | INFO | dns-server (http): request completed
latency_us = 12952
local_addr = [fd00:1122:3344:1::1]:5353
method = PUT
remote_addr = [fd00:1122:3344:1::2]:42550
req_id = 00235b80-0243-486b-84f4-955f2f993d2e
response_code = 204
uri = /config
|
| 21 | 2026-05-28T01:12:40.566Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 40174,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("0.pool.ntp.org."),
),
original: Query {
name: Name("0.pool.ntp.org."),
query_type: AAAA,
query_class: IN,
},
},
],
original: [
1,
48,
4,
112,
111,
111,
108,
3,
110,
116,
112,
3,
111,
114,
103,
0,
0,
28,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "0.pool.ntp.org."
peer_addr = [fd00:1122:3344:101::1b]:39539
req_id = aa5253f6-099a-454a-ac73-ecc3ff19b9a0
|
| 22 | 2026-05-28T01:12:40.579Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 59626,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("0.pool.ntp.org."),
),
original: Query {
name: Name("0.pool.ntp.org."),
query_type: AAAA,
query_class: IN,
},
},
],
original: [
1,
48,
4,
112,
111,
111,
108,
3,
110,
116,
112,
3,
111,
114,
103,
0,
0,
28,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "0.pool.ntp.org."
peer_addr = [fd00:1122:3344:101::1b]:36841
req_id = 6e765a17-cd94-4a66-8622-a01ac6deedda
|
| 23 | 2026-05-28T01:12:40.595Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 61696,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("0.pool.ntp.org."),
),
original: Query {
name: Name("0.pool.ntp.org."),
query_type: A,
query_class: IN,
},
},
],
original: [
1,
48,
4,
112,
111,
111,
108,
3,
110,
116,
112,
3,
111,
114,
103,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "0.pool.ntp.org."
peer_addr = [fd00:1122:3344:101::1b]:62432
req_id = 849c10ba-d2fb-4cbe-ab00-7e53c3819065
|
| 24 | 2026-05-28T01:13:58.581Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 64788,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:56952
req_id = bf961115-5321-4a38-a382-fec842f4af0c
|
| 25 | 2026-05-28T01:14:02.252Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 57029,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
),
original: Query {
name: Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
query_type: A,
query_class: IN,
},
},
],
original: [
51,
111,
120,
122,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
95,
97,
102,
101,
97,
51,
101,
50,
102,
45,
99,
99,
50,
51,
45,
52,
101,
100,
52,
45,
98,
57,
50,
99,
45,
48,
54,
98,
53,
55,
49,
99,
55,
50,
48,
98,
101,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."
peer_addr = [fd00:1122:3344:101::e]:57991
req_id = 088ec1a5-6353-454c-8a28-2fe66217c61a
|
| 26 | 2026-05-28T01:14:02.261Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 57029,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
),
original: Query {
name: Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
query_type: A,
query_class: IN,
},
},
],
original: [
51,
111,
120,
122,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
95,
97,
102,
101,
97,
51,
101,
50,
102,
45,
99,
99,
50,
51,
45,
52,
101,
100,
52,
45,
98,
57,
50,
99,
45,
48,
54,
98,
53,
55,
49,
99,
55,
50,
48,
98,
101,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."
peer_addr = [fd00:1122:3344:101::e]:60997
req_id = aeccc5e1-4309-4ccb-a034-73f003b11552
|
| 27 | 2026-05-28T01:14:04.684Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 16162,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("0.pool.ntp.org."),
),
original: Query {
name: Name("0.pool.ntp.org."),
query_type: AAAA,
query_class: IN,
},
},
],
original: [
1,
48,
4,
112,
111,
111,
108,
3,
110,
116,
112,
3,
111,
114,
103,
0,
0,
28,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "0.pool.ntp.org."
peer_addr = [fd00:1122:3344:101::1b]:56242
req_id = da742bbe-d3aa-463d-ae0e-a301ac88c286
|
| 28 | 2026-05-28T01:14:04.697Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 19655,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("0.pool.ntp.org."),
),
original: Query {
name: Name("0.pool.ntp.org."),
query_type: AAAA,
query_class: IN,
},
},
],
original: [
1,
48,
4,
112,
111,
111,
108,
3,
110,
116,
112,
3,
111,
114,
103,
0,
0,
28,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "0.pool.ntp.org."
peer_addr = [fd00:1122:3344:101::1b]:60551
req_id = e042d7eb-70e1-4a57-b520-8af0d9c6749b
|
| 29 | 2026-05-28T01:14:04.714Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 33218,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("0.pool.ntp.org."),
),
original: Query {
name: Name("0.pool.ntp.org."),
query_type: A,
query_class: IN,
},
},
],
original: [
1,
48,
4,
112,
111,
111,
108,
3,
110,
116,
112,
3,
111,
114,
103,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "0.pool.ntp.org."
peer_addr = [fd00:1122:3344:101::1b]:57695
req_id = 50a46bb6-7c5a-4b95-b2fc-e85fb2a80fff
|
| 30 | 2026-05-28T01:14:08.521Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 30309,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::a]:57478
req_id = aea32807-b197-4c85-b719-89a7580f2d9d
|
| 31 | 2026-05-28T01:14:08.555Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 5635,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:56196
req_id = 8b491970-a277-4bfc-bb40-57a282182667
|
| 32 | 2026-05-28T01:14:09.521Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 18965,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::b]:3033
req_id = f0befe98-68da-4ad8-82cf-ce82aaf8b4be
|
| 33 | 2026-05-28T01:14:09.794Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 41306,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::c]:37623
req_id = a762ce62-9d2c-4a1c-b769-aebaacdb8fa9
|
| 34 | 2026-05-28T01:14:18.559Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 34918,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:52235
req_id = 5709666b-4ef1-4fb5-8c20-ed12849e169f
|
| 35 | 2026-05-28T01:14:28.560Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 15468,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:52798
req_id = ad3ea098-9d65-410d-8d3e-0f1d5843a428
|
| 36 | 2026-05-28T01:14:38.563Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 25061,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:59459
req_id = 50fce9c2-7aeb-42c1-9c0a-c15fa6037c09
|
| 37 | 2026-05-28T01:14:47.319Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 45852,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::b]:21403
req_id = 6dc2b14c-9d7a-48d4-abc2-affb83f8480d
|
| 38 | 2026-05-28T01:14:47.716Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:1::1]:5353
remote_addr = [fd00:1122:3344:101::b]:56088
|
| 39 | 2026-05-28T01:14:47.794Z | INFO | dns-server (http): request completed
latency_us = 49999
local_addr = [fd00:1122:3344:1::1]:5353
method = GET
remote_addr = [fd00:1122:3344:101::b]:56088
req_id = 62287474-bc44-4fcb-a4df-2a2e186e3996
response_code = 200
uri = /config
|
| 40 | 2026-05-28T01:14:48.565Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 44589,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:57352
req_id = f6f03b05-ac3c-48ab-a72f-695a77d9f67c
|
| 41 | 2026-05-28T01:14:54.762Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 48580,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::b]:23709
req_id = ec650c05-f3ef-4c2c-a072-540039c2465e
|
| 42 | 2026-05-28T01:14:54.997Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:1::1]:5353
remote_addr = [fd00:1122:3344:101::b]:35361
|
| 43 | 2026-05-28T01:14:54.998Z | INFO | dns-server (http): request completed
latency_us = 478
local_addr = [fd00:1122:3344:1::1]:5353
method = GET
remote_addr = [fd00:1122:3344:101::b]:35361
req_id = 456c4cb9-190a-49bc-97fa-ee01b027ab2a
response_code = 200
uri = /config
|
| 44 | 2026-05-28T01:14:55.006Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:1::1]:5353
remote_addr = [fd00:1122:3344:101::b]:36409
|
| 45 | 2026-05-28T01:14:55.009Z | INFO | dns-server (store): attempting generation update
new_generation = 1
req_id = 6bf17517-2a07-4559-8b91-dd3ad1a44f16
|
| 46 | 2026-05-28T01:14:55.010Z | INFO | dns-server (store): updated generation
new_generation = 1
req_id = 6bf17517-2a07-4559-8b91-dd3ad1a44f16
|
| 47 | 2026-05-28T01:14:55.010Z | INFO | dns-server (http): request completed
latency_us = 3008
local_addr = [fd00:1122:3344:1::1]:5353
method = PUT
remote_addr = [fd00:1122:3344:101::b]:36409
req_id = 6bf17517-2a07-4559-8b91-dd3ad1a44f16
response_code = 204
uri = /config
|
| 48 | 2026-05-28T01:14:55.957Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 19524,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::b]:28560
req_id = ac5dde05-73f7-4d2f-b4f3-9b4d915d07ba
|
| 49 | 2026-05-28T01:14:56.084Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:1::1]:5353
remote_addr = [fd00:1122:3344:101::b]:60850
|
| 50 | 2026-05-28T01:14:56.085Z | INFO | dns-server (http): request completed
latency_us = 203
local_addr = [fd00:1122:3344:1::1]:5353
method = GET
remote_addr = [fd00:1122:3344:101::b]:60850
req_id = f37aa324-fe70-4086-9c1f-12284c14bd94
response_code = 200
uri = /config
|
| 51 | 2026-05-28T01:14:58.566Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 49766,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-cluster-native._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
26,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
99,
108,
117,
115,
116,
101,
114,
45,
110,
97,
116,
105,
118,
101,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::d]:65507
req_id = c65ed107-c2c4-4597-add5-b6b442360364
|
| 52 | 2026-05-28T01:14:59.327Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 40529,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::a]:46207
req_id = 19c079e5-e1d0-4abe-8290-ed62702ad420
|
| 53 | 2026-05-28T01:14:59.536Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:1::1]:5353
remote_addr = [fd00:1122:3344:101::a]:44378
|
| 54 | 2026-05-28T01:14:59.536Z | INFO | dns-server (http): request completed
latency_us = 186
local_addr = [fd00:1122:3344:1::1]:5353
method = GET
remote_addr = [fd00:1122:3344:101::a]:44378
req_id = 397bdfca-47e6-493e-8e29-a22937326118
response_code = 200
uri = /config
|
| 55 | 2026-05-28T01:15:00.326Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 35472,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4."),
),
original: Query {
name: Name("oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
54,
50,
99,
102,
54,
48,
57,
56,
45,
102,
50,
51,
54,
45,
52,
99,
52,
100,
45,
98,
97,
53,
50,
45,
54,
100,
99,
98,
55,
53,
48,
49,
101,
55,
102,
52,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4."
peer_addr = [fd00:1122:3344:101::6]:34670
req_id = a1e25cc2-2e27-48d0-b0b0-2cbd8041b4cd
|
| 56 | 2026-05-28T01:15:00.327Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 35472,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4."),
),
original: Query {
name: Name("oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
54,
50,
99,
102,
54,
48,
57,
56,
45,
102,
50,
51,
54,
45,
52,
99,
52,
100,
45,
98,
97,
53,
50,
45,
54,
100,
99,
98,
55,
53,
48,
49,
101,
55,
102,
52,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4."
peer_addr = [fd00:1122:3344:101::6]:47641
req_id = 2c124fb5-fbaf-4301-8c01-b621b4d754e3
|
| 57 | 2026-05-28T01:15:00.385Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 55261,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_switch."),
),
original: Query {
name: Name("oxz_switch."),
query_type: A,
query_class: IN,
},
},
],
original: [
10,
111,
120,
122,
95,
115,
119,
105,
116,
99,
104,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_switch."
peer_addr = [fd00:1122:3344:101::2]:36722
req_id = 767fbb4e-67af-4c0a-b626-174106383563
|
| 58 | 2026-05-28T01:15:00.385Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 55261,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_switch."),
),
original: Query {
name: Name("oxz_switch."),
query_type: A,
query_class: IN,
},
},
],
original: [
10,
111,
120,
122,
95,
115,
119,
105,
116,
99,
104,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_switch."
peer_addr = [fd00:1122:3344:101::2]:34587
req_id = 990f4505-6e51-4beb-9fb4-f45adc700e4e
|
| 59 | 2026-05-28T01:15:00.508Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 28629,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
),
original: Query {
name: Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
query_type: A,
query_class: IN,
},
},
],
original: [
51,
111,
120,
122,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
95,
97,
102,
101,
97,
51,
101,
50,
102,
45,
99,
99,
50,
51,
45,
52,
101,
100,
52,
45,
98,
57,
50,
99,
45,
48,
54,
98,
53,
55,
49,
99,
55,
50,
48,
98,
101,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."
peer_addr = [fd00:1122:3344:101::e]:47585
req_id = 018738c5-a330-4dd0-9b07-ea14a6840613
|
| 60 | 2026-05-28T01:15:00.509Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 28629,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
),
original: Query {
name: Name("oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."),
query_type: A,
query_class: IN,
},
},
],
original: [
51,
111,
120,
122,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
95,
97,
102,
101,
97,
51,
101,
50,
102,
45,
99,
99,
50,
51,
45,
52,
101,
100,
52,
45,
98,
57,
50,
99,
45,
48,
54,
98,
53,
55,
49,
99,
55,
50,
48,
98,
101,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be."
peer_addr = [fd00:1122:3344:101::e]:55207
req_id = 778bace9-3c24-4736-a1bf-bfb168bd95f1
|
| 61 | 2026-05-28T01:15:00.516Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 32243,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2."),
),
original: Query {
name: Name("oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
55,
99,
48,
101,
97,
49,
52,
97,
45,
97,
48,
48,
49,
45,
52,
102,
48,
101,
45,
97,
57,
50,
52,
45,
49,
54,
100,
52,
50,
51,
51,
101,
100,
98,
101,
50,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2."
peer_addr = [fd00:1122:3344:101::4]:52231
req_id = 93ed5b76-c01f-461c-9c2f-afe7bd8c871e
|
| 62 | 2026-05-28T01:15:00.516Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 32243,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2."),
),
original: Query {
name: Name("oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
55,
99,
48,
101,
97,
49,
52,
97,
45,
97,
48,
48,
49,
45,
52,
102,
48,
101,
45,
97,
57,
50,
52,
45,
49,
54,
100,
52,
50,
51,
51,
101,
100,
98,
101,
50,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2."
peer_addr = [fd00:1122:3344:101::4]:59230
req_id = 910d2f11-384b-4be3-add1-4992ebac2c0f
|
| 63 | 2026-05-28T01:15:00.523Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 1319,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 1,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
),
original: Query {
name: Name("_clickhouse-admin-keeper._tcp.control-plane.oxide.internal."),
query_type: SRV,
query_class: IN,
},
},
],
original: [
24,
95,
99,
108,
105,
99,
107,
104,
111,
117,
115,
101,
45,
97,
100,
109,
105,
110,
45,
107,
101,
101,
112,
101,
114,
4,
95,
116,
99,
112,
13,
99,
111,
110,
116,
114,
111,
108,
45,
112,
108,
97,
110,
101,
5,
111,
120,
105,
100,
101,
8,
105,
110,
116,
101,
114,
110,
97,
108,
0,
0,
33,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: Some(
Edns {
rcode_high: 0,
version: 0,
flags: EdnsFlags {
dnssec_ok: false,
z: 0,
},
max_payload: 1232,
options: OPT {
options: [],
},
},
),
} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal
peer_addr = [fd00:1122:3344:101::c]:31621
req_id = 6d8bdb70-56ba-460d-9a09-d0eaff13b7df
|
| 64 | 2026-05-28T01:15:00.536Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 47615,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee."),
),
original: Query {
name: Name("oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
48,
99,
98,
52,
101,
56,
101,
102,
45,
54,
56,
97,
53,
45,
52,
56,
100,
98,
45,
56,
55,
54,
100,
45,
99,
102,
49,
97,
98,
101,
49,
101,
50,
56,
101,
101,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee."
peer_addr = [fd00:1122:3344:101::3]:33682
req_id = 89d59ed7-6cd0-4dca-9ea0-5d5144096ee5
|
| 65 | 2026-05-28T01:15:00.536Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 47615,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee."),
),
original: Query {
name: Name("oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
48,
99,
98,
52,
101,
56,
101,
102,
45,
54,
56,
97,
53,
45,
52,
56,
100,
98,
45,
56,
55,
54,
100,
45,
99,
102,
49,
97,
98,
101,
49,
101,
50,
56,
101,
101,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee."
peer_addr = [fd00:1122:3344:101::3]:55331
req_id = a2644b7e-1535-41ee-9303-0f2cf41c2133
|
| 66 | 2026-05-28T01:15:00.544Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 4239,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126."),
),
original: Query {
name: Name("oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
57,
55,
48,
51,
97,
97,
48,
53,
45,
53,
51,
54,
98,
45,
52,
51,
49,
48,
45,
56,
100,
52,
56,
45,
55,
57,
52,
98,
100,
57,
102,
102,
56,
49,
50,
54,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126."
peer_addr = [fd00:1122:3344:101::7]:35875
req_id = 553714ea-0804-4a51-ae77-2c8d28c3953b
|
| 67 | 2026-05-28T01:15:00.544Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 4239,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126."),
),
original: Query {
name: Name("oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
57,
55,
48,
51,
97,
97,
48,
53,
45,
53,
51,
54,
98,
45,
52,
51,
49,
48,
45,
56,
100,
52,
56,
45,
55,
57,
52,
98,
100,
57,
102,
102,
56,
49,
50,
54,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126."
peer_addr = [fd00:1122:3344:101::7]:43756
req_id = 3473b7ce-2751-4024-a214-4cfa21f9e467
|
| 68 | 2026-05-28T01:15:00.564Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 62175,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30."),
),
original: Query {
name: Name("oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
51,
49,
57,
49,
98,
55,
52,
48,
45,
55,
100,
99,
55,
45,
52,
52,
51,
100,
45,
56,
99,
97,
48,
45,
100,
51,
55,
102,
99,
49,
99,
101,
101,
99,
51,
48,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30."
peer_addr = [fd00:1122:3344:101::5]:45091
req_id = 31d344b1-61c4-4f99-bffa-108d9de6bca3
|
| 69 | 2026-05-28T01:15:00.565Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 62175,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30."),
),
original: Query {
name: Name("oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30."),
query_type: A,
query_class: IN,
},
},
],
original: [
52,
111,
120,
122,
95,
99,
111,
99,
107,
114,
111,
97,
99,
104,
100,
98,
95,
51,
49,
57,
49,
98,
55,
52,
48,
45,
55,
100,
99,
55,
45,
52,
52,
51,
100,
45,
56,
99,
97,
48,
45,
100,
51,
55,
102,
99,
49,
99,
101,
101,
99,
51,
48,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30."
peer_addr = [fd00:1122:3344:101::5]:62513
req_id = ac469cdf-48ab-42aa-9817-f5b227085519
|
| 70 | 2026-05-28T01:15:00.652Z | INFO | dns-server (http): accepted connection
local_addr = [fd00:1122:3344:1::1]:5353
remote_addr = [fd00:1122:3344:101::c]:46383
|
| 71 | 2026-05-28T01:15:00.653Z | INFO | dns-server (http): request completed
latency_us = 258
local_addr = [fd00:1122:3344:1::1]:5353
method = GET
remote_addr = [fd00:1122:3344:101::c]:46383
req_id = 5112c6f9-d4bc-4b75-b2f9-87db86f0213c
response_code = 200
uri = /config
|
| 72 | 2026-05-28T01:15:00.919Z | ERRO | dns-server (dns): failed to handle incoming DNS message: MessageRequest {
header: Header {
id: 39768,
message_type: Query,
op_code: Query,
authoritative: false,
truncation: false,
recursion_desired: true,
recursion_available: false,
authentic_data: false,
checking_disabled: false,
response_code: NoError,
query_count: 1,
answer_count: 0,
name_server_count: 0,
additional_count: 0,
},
queries: Queries {
queries: [
LowerQuery {
name: LowerName(
Name("oxz_ntp_232ef14c-e91e-40df-9b38-b5764fde289c."),
),
original: Query {
name: Name("oxz_ntp_232ef14c-e91e-40df-9b38-b5764fde289c."),
query_type: A,
query_class: IN,
},
},
],
original: [
44,
111,
120,
122,
95,
110,
116,
112,
95,
50,
51,
50,
101,
102,
49,
52,
99,
45,
101,
57,
49,
101,
45,
52,
48,
100,
102,
45,
57,
98,
51,
56,
45,
98,
53,
55,
54,
52,
102,
100,
101,
50,
56,
57,
99,
0,
0,
1,
0,
1,
],
},
answers: [],
name_servers: [],
additionals: [],
sig0: [],
edns: None,
} SERVFAIL: server is not authoritative for name: "oxz_ntp_232ef14c-e91e-40df-9b38-b5764fde289c."
peer_addr = [fd00:1122:3344:101::1b]:54926
req_id = d3e48773-d499-42b5-a107-902de99dd044
|