[ May 28 01:12:26 Enabled. ] [ May 28 01:12:26 Rereading configuration. ] [ May 28 01:12:26 Rereading configuration. ] [ May 28 01:12:27 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:2::1]:5353 --dns-address [fd00:1122:3344:2::1]:53 &"). ] [ May 28 01:12:28 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:12:28.018197734Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"dns_config":"Config { bind_address: [fd00:1122:3344:2::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:12:28.018364888Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:12:28.041242375Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:12:28.041262745Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:12:28.041323697Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"component":"dns","local_address":"[fd00:1122:3344:2::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:12:28.041625082Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40212,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:12:28.042353656Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::2]:15664","req_id":"cab93ffa-4446-4861-a01d-77640eacb3c5","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:12:28.991842974Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:2::2]:61645"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:12:28.992493156Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"new_generation":1,"req_id":"292ea58d-a18e-40cb-b3fb-45b7cfd5ce6f","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:12:28.992528357Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:12:29.005335806Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:12:29.005364477Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"new_generation":1,"req_id":"292ea58d-a18e-40cb-b3fb-45b7cfd5ce6f","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:12:29.005403158Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"uri":"/config","method":"PUT","req_id":"292ea58d-a18e-40cb-b3fb-45b7cfd5ce6f","remote_addr":"[fd00:1122:3344:2::2]:61645","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":13226,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40174,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:12:40.567119539Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::1b]:45320","req_id":"64ed1a46-8516-4e56-85dc-91d731234d11","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59626,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:12:40.579626143Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::1b]:51270","req_id":"abf33ba6-07c1-4815-b555-ff6864d2bfa1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 61696,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:12:40.595569861Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::1b]:34384","req_id":"4d2834ec-b8dc-4bb7-a087-65aeaeaa4e6a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43915,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:13:58.579417813Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::d]:63083","req_id":"70327e39-413e-49dd-a9c4-bf2dc293d4e9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57029,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 97,\n 102,\n 101,\n 97,\n 51,\n 101,\n 50,\n 102,\n 45,\n 99,\n 99,\n 50,\n 51,\n 45,\n 52,\n 101,\n 100,\n 52,\n 45,\n 98,\n 57,\n 50,\n 99,\n 45,\n 48,\n 54,\n 98,\n 53,\n 55,\n 49,\n 99,\n 55,\n 50,\n 48,\n 98,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:14:02.255750571Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::e]:63798","req_id":"81e1a7c6-5bce-44d8-86d8-ac2f25455e0e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57029,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 97,\n 102,\n 101,\n 97,\n 51,\n 101,\n 50,\n 102,\n 45,\n 99,\n 99,\n 50,\n 51,\n 45,\n 52,\n 101,\n 100,\n 52,\n 45,\n 98,\n 57,\n 50,\n 99,\n 45,\n 48,\n 54,\n 98,\n 53,\n 55,\n 49,\n 99,\n 55,\n 50,\n 48,\n 98,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:14:02.261411237Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::e]:33874","req_id":"f7a08455-0707-4b19-b155-b5ea0da7cfa1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16162,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:14:04.684782244Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::1b]:61572","req_id":"a4e4f465-18b2-4c4b-b3c7-4c43b1a9f2ab","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 19655,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:14:04.698139714Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::1b]:63339","req_id":"8a81c678-3ed7-4e0d-92a1-47dad34a88d4","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 33218,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:14:04.71501751Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::1b]:44427","req_id":"883ad9ae-593d-4035-b6fe-8fe2bd985421","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 51875,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:14:08.521168013Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::a]:46148","req_id":"cdc1f102-b834-4387-82ed-deab236295dc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 12555,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:14:08.5557605Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::d]:56508","req_id":"bf184498-9174-4597-afd3-5658ac359497","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18963,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:14:09.520890759Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::b]:13081","req_id":"66a41323-ea9e-40bf-8117-d3ee1ce1ac62","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 19782,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:14:09.79441115Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::c]:39501","req_id":"d636a85c-cec6-41d5-bc91-c330626d9cab","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 19073,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:14:18.559415439Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::d]:59599","req_id":"032f0bb2-fcaf-4092-85c0-4cae9fc7f58b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54851,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:14:28.560977536Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::d]:61665","req_id":"febb7c85-bf10-45b2-805e-33d8dad2da5c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21191,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:14:38.563389181Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::d]:58756","req_id":"817a06ce-0917-44b9-a5e5-6048b2f8500c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 12035,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:14:47.320487791Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::b]:31578","req_id":"79995bf1-d133-47b8-b32a-a53788966f68","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:14:47.629478257Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:34622"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:14:47.705291234Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"uri":"/config","method":"GET","req_id":"7de6c706-eef5-4b84-a20e-92f3e741c70c","remote_addr":"[fd00:1122:3344:101::b]:34622","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":48406,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14477,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:14:48.565538604Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::d]:59613","req_id":"8a3f2127-ee8a-4ff8-8bc5-85e84ce1fa0d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 22445,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:14:54.762876406Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::b]:8863","req_id":"a517003b-16c6-41d0-9b78-1594f9a8014d","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:14:54.996691297Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:42486"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:14:54.996967002Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"uri":"/config","method":"GET","req_id":"eb3aeb89-c203-44d8-9ba6-1584105a779f","remote_addr":"[fd00:1122:3344:101::b]:42486","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":195,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:14:55.00594077Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:54139"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:14:55.009885183Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"new_generation":1,"req_id":"cb512685-476c-400e-8358-1ca1c0f15840","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:14:55.009942615Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"new_generation":1,"req_id":"cb512685-476c-400e-8358-1ca1c0f15840","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:14:55.009950415Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"uri":"/config","method":"PUT","req_id":"cb512685-476c-400e-8358-1ca1c0f15840","remote_addr":"[fd00:1122:3344:101::b]:54139","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":2959,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23564,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:14:55.957972986Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::b]:15597","req_id":"a61e60c8-eecd-4654-a7f1-4433599b4942","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:14:56.084252276Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:63720"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:14:56.084508241Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"uri":"/config","method":"GET","req_id":"acc4b68e-4605-4079-a1d3-9720d24c1499","remote_addr":"[fd00:1122:3344:101::b]:63720","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":186,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 19785,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:14:58.566419354Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::d]:63573","req_id":"879ad2d9-8264-46bd-ab04-40fb218d29bd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 47794,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:14:59.327935128Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::a]:52223","req_id":"e5aec55e-51d1-4ec2-ac19-c6f11fca9eea","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:14:59.535261723Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:36863"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:14:59.535540618Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"uri":"/config","method":"GET","req_id":"bb124f4f-7787-4a6b-8460-cc9f0725646b","remote_addr":"[fd00:1122:3344:101::a]:36863","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":225,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 35472,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 50,\n 99,\n 102,\n 54,\n 48,\n 57,\n 56,\n 45,\n 102,\n 50,\n 51,\n 54,\n 45,\n 52,\n 99,\n 52,\n 100,\n 45,\n 98,\n 97,\n 53,\n 50,\n 45,\n 54,\n 100,\n 99,\n 98,\n 55,\n 53,\n 48,\n 49,\n 101,\n 55,\n 102,\n 52,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:15:00.326980182Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::6]:37171","req_id":"dfd1bcb3-56da-4c70-8204-a9f8e536dc0c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 35472,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 50,\n 99,\n 102,\n 54,\n 48,\n 57,\n 56,\n 45,\n 102,\n 50,\n 51,\n 54,\n 45,\n 52,\n 99,\n 52,\n 100,\n 45,\n 98,\n 97,\n 53,\n 50,\n 45,\n 54,\n 100,\n 99,\n 98,\n 55,\n 53,\n 48,\n 49,\n 101,\n 55,\n 102,\n 52,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_62cf6098-f236-4c4d-ba52-6dcb7501e7f4.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:15:00.327604394Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::6]:63935","req_id":"a537147c-d7dc-4c60-a023-7dd17c5e021e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 55261,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:15:00.385531957Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::2]:55117","req_id":"5f1fe00c-f2eb-467d-8315-e2fd36a28a2c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 55261,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:15:00.386184499Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::2]:48432","req_id":"72a59464-277a-449f-8ddf-3bf63665a144","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 28629,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 97,\n 102,\n 101,\n 97,\n 51,\n 101,\n 50,\n 102,\n 45,\n 99,\n 99,\n 50,\n 51,\n 45,\n 52,\n 101,\n 100,\n 52,\n 45,\n 98,\n 57,\n 50,\n 99,\n 45,\n 48,\n 54,\n 98,\n 53,\n 55,\n 49,\n 99,\n 55,\n 50,\n 48,\n 98,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:15:00.509109887Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::e]:63573","req_id":"dfe21dc9-1722-42e2-a661-d019e9354ee0","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 28629,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 97,\n 102,\n 101,\n 97,\n 51,\n 101,\n 50,\n 102,\n 45,\n 99,\n 99,\n 50,\n 51,\n 45,\n 52,\n 101,\n 100,\n 52,\n 45,\n 98,\n 57,\n 50,\n 99,\n 45,\n 48,\n 54,\n 98,\n 53,\n 55,\n 49,\n 99,\n 55,\n 50,\n 48,\n 98,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_afea3e2f-cc23-4ed4-b92c-06b571c720be.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:15:00.509787749Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::e]:56741","req_id":"2a575fd5-bab9-4a9d-a9f0-910a05f3954a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 32243,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 55,\n 99,\n 48,\n 101,\n 97,\n 49,\n 52,\n 97,\n 45,\n 97,\n 48,\n 48,\n 49,\n 45,\n 52,\n 102,\n 48,\n 101,\n 45,\n 97,\n 57,\n 50,\n 52,\n 45,\n 49,\n 54,\n 100,\n 52,\n 50,\n 51,\n 51,\n 101,\n 100,\n 98,\n 101,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:15:00.516305331Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::4]:52845","req_id":"eefbdc62-2248-44c2-8bf1-006cb52451ad","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 32243,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 55,\n 99,\n 48,\n 101,\n 97,\n 49,\n 52,\n 97,\n 45,\n 97,\n 48,\n 48,\n 49,\n 45,\n 52,\n 102,\n 48,\n 101,\n 45,\n 97,\n 57,\n 50,\n 52,\n 45,\n 49,\n 54,\n 100,\n 52,\n 50,\n 51,\n 51,\n 101,\n 100,\n 98,\n 101,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_7c0ea14a-a001-4f0e-a924-16d4233edbe2.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:15:00.517003234Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::4]:57139","req_id":"f21dd05b-6b11-4ea8-83d0-af1cdb4d611c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1788,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:15:00.523266921Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::c]:22884","req_id":"2832c690-1ced-4226-ad24-a883145e107d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 47615,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 48,\n 99,\n 98,\n 52,\n 101,\n 56,\n 101,\n 102,\n 45,\n 54,\n 56,\n 97,\n 53,\n 45,\n 52,\n 56,\n 100,\n 98,\n 45,\n 56,\n 55,\n 54,\n 100,\n 45,\n 99,\n 102,\n 49,\n 97,\n 98,\n 101,\n 49,\n 101,\n 50,\n 56,\n 101,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:15:00.536382536Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::3]:52878","req_id":"9e0639cc-730d-4e49-ab36-b7955f541790","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 47615,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 48,\n 99,\n 98,\n 52,\n 101,\n 56,\n 101,\n 102,\n 45,\n 54,\n 56,\n 97,\n 53,\n 45,\n 52,\n 56,\n 100,\n 98,\n 45,\n 56,\n 55,\n 54,\n 100,\n 45,\n 99,\n 102,\n 49,\n 97,\n 98,\n 101,\n 49,\n 101,\n 50,\n 56,\n 101,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_0cb4e8ef-68a5-48db-876d-cf1abe1e28ee.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:15:00.53710846Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::3]:47751","req_id":"1bbf9c06-bd4b-4e8a-8b34-d53fcf2afdfb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4239,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 57,\n 55,\n 48,\n 51,\n 97,\n 97,\n 48,\n 53,\n 45,\n 53,\n 51,\n 54,\n 98,\n 45,\n 52,\n 51,\n 49,\n 48,\n 45,\n 56,\n 100,\n 52,\n 56,\n 45,\n 55,\n 57,\n 52,\n 98,\n 100,\n 57,\n 102,\n 102,\n 56,\n 49,\n 50,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:15:00.544458747Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::7]:52876","req_id":"36db5820-b407-495d-ad01-4fae75623819","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4239,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 57,\n 55,\n 48,\n 51,\n 97,\n 97,\n 48,\n 53,\n 45,\n 53,\n 51,\n 54,\n 98,\n 45,\n 52,\n 51,\n 49,\n 48,\n 45,\n 56,\n 100,\n 52,\n 56,\n 45,\n 55,\n 57,\n 52,\n 98,\n 100,\n 57,\n 102,\n 102,\n 56,\n 49,\n 50,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_9703aa05-536b-4310-8d48-794bd9ff8126.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:15:00.545081029Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::7]:61241","req_id":"76f05d7d-828b-4053-8882-f6563265f4e1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62175,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 49,\n 57,\n 49,\n 98,\n 55,\n 52,\n 48,\n 45,\n 55,\n 100,\n 99,\n 55,\n 45,\n 52,\n 52,\n 51,\n 100,\n 45,\n 56,\n 99,\n 97,\n 48,\n 45,\n 100,\n 51,\n 55,\n 102,\n 99,\n 49,\n 99,\n 101,\n 101,\n 99,\n 51,\n 48,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:15:00.565190595Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::5]:52188","req_id":"3a1e97aa-5e88-4f6d-b183-8837dcd83f7e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62175,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 49,\n 57,\n 49,\n 98,\n 55,\n 52,\n 48,\n 45,\n 55,\n 100,\n 99,\n 55,\n 45,\n 52,\n 52,\n 51,\n 100,\n 45,\n 56,\n 99,\n 97,\n 48,\n 45,\n 100,\n 51,\n 55,\n 102,\n 99,\n 49,\n 99,\n 101,\n 101,\n 99,\n 51,\n 48,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_3191b740-7dc7-443d-8ca0-d37fc1ceec30.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:15:00.565914158Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::5]:54030","req_id":"11ca84ec-1912-468b-a9d0-16c724e40dc0","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:15:00.652042928Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:59220"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-28T01:15:00.652412755Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"uri":"/config","method":"GET","req_id":"2edbe72a-bf6b-4495-ab51-e2bbd50a78d2","remote_addr":"[fd00:1122:3344:101::c]:59220","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":236,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 39768,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_232ef14c-e91e-40df-9b38-b5764fde289c.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_232ef14c-e91e-40df-9b38-b5764fde289c.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 50,\n 51,\n 50,\n 101,\n 102,\n 49,\n 52,\n 99,\n 45,\n 101,\n 57,\n 49,\n 101,\n 45,\n 52,\n 48,\n 100,\n 102,\n 45,\n 57,\n 98,\n 51,\n 56,\n 45,\n 98,\n 53,\n 55,\n 54,\n 52,\n 102,\n 100,\n 101,\n 50,\n 56,\n 57,\n 99,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_232ef14c-e91e-40df-9b38-b5764fde289c.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-28T01:15:00.919378805Z","hostname":"oxz_internal_dns_4e9cab4a-7313-4c84-9b24-f38603c9dacd","pid":5436,"peer_addr":"[fd00:1122:3344:101::1b]:64509","req_id":"fdac4e91-0148-4c7b-b53f-2935d13e47fb","component":"dns"}