[ Mar 24 22:26:17 Enabled. ] [ Mar 24 22:26:17 Rereading configuration. ] [ Mar 24 22:26:18 Rereading configuration. ] [ Mar 24 22:26:20 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:2::1]:5353 --dns-address [fd00:1122:3344:2::1]:53 &"). ] [ Mar 24 22:26:20 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:26:20.382022371Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"dns_config":"Config { bind_address: [fd00:1122:3344:2::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:26:20.382233847Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:26:20.402321554Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:26:20.402349065Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:26:20.402396536Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"component":"dns","local_address":"[fd00:1122:3344:2::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:26:20.402669995Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:26:20.419300476Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:2::2]:34033"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:26:20.420148412Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"new_generation":1,"req_id":"bf42fdfe-4c2d-4a81-b6ec-8ffa562cb757","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:26:20.420160892Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:26:20.462664748Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:26:20.462707299Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"new_generation":1,"req_id":"bf42fdfe-4c2d-4a81-b6ec-8ffa562cb757","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:26:20.46274372Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"uri":"/config","method":"PUT","req_id":"bf42fdfe-4c2d-4a81-b6ec-8ffa562cb757","remote_addr":"[fd00:1122:3344:2::2]:34033","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":43012,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46131,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:26:32.75398778Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::1b]:39373","req_id":"4a56c8d5-da95-4df5-9ed3-cb24dde87ac3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52454,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:26:32.758856649Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::1b]:49015","req_id":"a66adf05-58ef-4521-912c-d81f40dd2e07","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 17627,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:26:32.76310788Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::1b]:59164","req_id":"537442b5-48de-42b2-a86b-114e13699eef","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38273,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:27:08.539718943Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::1b]:57333","req_id":"695a5e4e-375f-411c-817c-8c56593386ef","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 15945,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:27:08.546047278Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::1b]:35311","req_id":"3feb47ae-f2dd-4fe1-9c68-a2c20474feb9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4295,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:27:08.550881767Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::1b]:65166","req_id":"86593d9d-ee56-419b-a8a2-8838bdb04b44","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20486,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:28:04.559809058Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::1b]:36876","req_id":"6333d21e-f7f1-4092-9a55-9981b19eeebf","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20303,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:28:04.5643954Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::1b]:52967","req_id":"93dc93b3-6d63-40fe-91ed-f177de137baa","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 29774,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:28:04.56829203Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::1b]:56357","req_id":"47d1b7c2-f488-4dac-9341-d03b3a6a676a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 47368,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:28:33.31456168Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::1b]:33339","req_id":"970c49f0-59f0-4696-80d6-22cbdb889ad1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1333,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:28:33.581390736Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::1b]:41081","req_id":"8ab8b9d8-82ee-4f1d-8136-658f91e64e58","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 27592,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:28:33.586111081Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::1b]:61868","req_id":"ba5f521f-909a-429f-b082-c986d13f6b50","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 28404,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:28:56.742700197Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::d]:49219","req_id":"82147d9e-4023-4865-863e-f60827f39030","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 5654,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:29:06.193637381Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::a]:65428","req_id":"03c6352d-9ca6-4401-aca5-e27c4f807305","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 24329,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:29:06.366753519Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::c]:21167","req_id":"288e54f7-aab5-4c16-94ab-392e83f28ba2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16100,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:29:06.743757149Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::d]:65061","req_id":"aab0a5c7-1770-42f8-9ffb-1f9ebb72afcf","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54655,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:29:06.755064537Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::b]:52729","req_id":"ece525b4-1772-4773-8350-bf57f0f12b47","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 10884,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_c04272af-e5af-4dd3-be84-797a9108d0ab.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_c04272af-e5af-4dd3-be84-797a9108d0ab.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 99,\n 48,\n 52,\n 50,\n 55,\n 50,\n 97,\n 102,\n 45,\n 101,\n 53,\n 97,\n 102,\n 45,\n 52,\n 100,\n 100,\n 51,\n 45,\n 98,\n 101,\n 56,\n 52,\n 45,\n 55,\n 57,\n 55,\n 97,\n 57,\n 49,\n 48,\n 56,\n 100,\n 48,\n 97,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_c04272af-e5af-4dd3-be84-797a9108d0ab.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:29:10.080651117Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::e]:59400","req_id":"cdcc3dde-dc73-49f8-a1eb-45cb1903b433","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 10884,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_c04272af-e5af-4dd3-be84-797a9108d0ab.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_c04272af-e5af-4dd3-be84-797a9108d0ab.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 99,\n 48,\n 52,\n 50,\n 55,\n 50,\n 97,\n 102,\n 45,\n 101,\n 53,\n 97,\n 102,\n 45,\n 52,\n 100,\n 100,\n 51,\n 45,\n 98,\n 101,\n 56,\n 52,\n 45,\n 55,\n 57,\n 55,\n 97,\n 57,\n 49,\n 48,\n 56,\n 100,\n 48,\n 97,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_c04272af-e5af-4dd3-be84-797a9108d0ab.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:29:10.081134682Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::e]:42120","req_id":"f3411346-02c5-460e-a0ba-bea43d6d13ec","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42971,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:29:16.74463105Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::d]:61230","req_id":"4c03af85-e3f8-4515-8958-ad338348d8cd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 33391,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:29:26.746780619Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::d]:52262","req_id":"cdcd682a-0760-4d5f-b92e-ec4b633b62fe","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 8943,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:29:36.748318169Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::d]:50973","req_id":"d9bc0fb5-0412-41aa-a640-f650db4aa8c2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62619,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:29:40.057567604Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::c]:57964","req_id":"24be59a0-3719-4d93-ad2f-a897ac7e8020","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:29:40.415482818Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:45597"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:29:40.472571011Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"uri":"/config","method":"GET","req_id":"45ee5d2f-84ef-4c3e-a166-8e9b0ddbc421","remote_addr":"[fd00:1122:3344:101::c]:45597","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":29887,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:29:42.945044602Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:33422"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:29:42.947661312Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"new_generation":1,"req_id":"a8d10477-e962-4644-83d6-60a4a9fc1e18","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:29:42.947683312Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"new_generation":1,"req_id":"a8d10477-e962-4644-83d6-60a4a9fc1e18","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:29:42.947736434Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"uri":"/config","method":"PUT","req_id":"a8d10477-e962-4644-83d6-60a4a9fc1e18","remote_addr":"[fd00:1122:3344:101::a]:33422","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":1064,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 39673,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:29:42.967759199Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::a]:29492","req_id":"ef497789-c206-4290-ad88-9f2c86b4cd48","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:29:43.216788558Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:36253"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:29:43.217052197Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"uri":"/config","method":"GET","req_id":"dcf82a1c-ecc7-4b7c-9c85-5696467d993f","remote_addr":"[fd00:1122:3344:101::a]:36253","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":207,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34564,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:29:43.692921815Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::a]:53014","req_id":"4e2d4c1e-164c-4d28-9099-f1013d82bf54","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:29:43.81094739Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:45969"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:29:43.811230549Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"uri":"/config","method":"GET","req_id":"5e72c4c2-613b-46c3-b886-b70e01609bd1","remote_addr":"[fd00:1122:3344:101::a]:45969","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":219,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 44377,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:29:46.748475126Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::d]:55121","req_id":"b3a25fc4-788b-4118-9fab-78af8bc95c11","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20103,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:29:47.056382434Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::c]:59077","req_id":"d11e04d7-7cf7-4670-816f-d146aa072680","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:29:47.177997869Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:35729"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:29:47.178449392Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"uri":"/config","method":"GET","req_id":"0814ef56-896d-4681-acff-8e03acd887a2","remote_addr":"[fd00:1122:3344:101::c]:35729","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":372,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 56009,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:29:47.600375603Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::b]:11230","req_id":"ca0a716c-bbf7-4da0-a221-259e064d854d","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:29:47.765192366Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:44892"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T22:29:47.765537226Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"uri":"/config","method":"GET","req_id":"396eb6fb-b2f4-4cd3-a83f-5e8d0ed1d70b","remote_addr":"[fd00:1122:3344:101::b]:44892","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":258,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 736,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:29:56.749212Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::d]:56843","req_id":"45b96310-3464-4c49-bb17-8eff0e48f232","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 58599,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_5aa4ff3e-3c75-4822-ba8a-0f1f88912467.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_5aa4ff3e-3c75-4822-ba8a-0f1f88912467.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 53,\n 97,\n 97,\n 52,\n 102,\n 102,\n 51,\n 101,\n 45,\n 51,\n 99,\n 55,\n 53,\n 45,\n 52,\n 56,\n 50,\n 50,\n 45,\n 98,\n 97,\n 56,\n 97,\n 45,\n 48,\n 102,\n 49,\n 102,\n 56,\n 56,\n 57,\n 49,\n 50,\n 52,\n 54,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_5aa4ff3e-3c75-4822-ba8a-0f1f88912467.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:30:00.191062587Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::7]:33754","req_id":"df8c56e8-2f93-48fb-872d-010e2aa37984","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 58599,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_5aa4ff3e-3c75-4822-ba8a-0f1f88912467.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_5aa4ff3e-3c75-4822-ba8a-0f1f88912467.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 53,\n 97,\n 97,\n 52,\n 102,\n 102,\n 51,\n 101,\n 45,\n 51,\n 99,\n 55,\n 53,\n 45,\n 52,\n 56,\n 50,\n 50,\n 45,\n 98,\n 97,\n 56,\n 97,\n 45,\n 48,\n 102,\n 49,\n 102,\n 56,\n 56,\n 57,\n 49,\n 50,\n 52,\n 54,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_5aa4ff3e-3c75-4822-ba8a-0f1f88912467.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:30:00.191666726Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::7]:32931","req_id":"fa322e45-4ac6-4b1e-8f9a-64431e361da8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 61273,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_608d6493-6f0c-4ee6-bf03-c5c0e466c178.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_608d6493-6f0c-4ee6-bf03-c5c0e466c178.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 48,\n 56,\n 100,\n 54,\n 52,\n 57,\n 51,\n 45,\n 54,\n 102,\n 48,\n 99,\n 45,\n 52,\n 101,\n 101,\n 54,\n 45,\n 98,\n 102,\n 48,\n 51,\n 45,\n 99,\n 53,\n 99,\n 48,\n 101,\n 52,\n 54,\n 54,\n 99,\n 49,\n 55,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_608d6493-6f0c-4ee6-bf03-c5c0e466c178.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:30:00.269156806Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::5]:43761","req_id":"2ebbe860-0e0c-4e7a-b668-be0122201562","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 61273,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_608d6493-6f0c-4ee6-bf03-c5c0e466c178.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_608d6493-6f0c-4ee6-bf03-c5c0e466c178.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 54,\n 48,\n 56,\n 100,\n 54,\n 52,\n 57,\n 51,\n 45,\n 54,\n 102,\n 48,\n 99,\n 45,\n 52,\n 101,\n 101,\n 54,\n 45,\n 98,\n 102,\n 48,\n 51,\n 45,\n 99,\n 53,\n 99,\n 48,\n 101,\n 52,\n 54,\n 54,\n 99,\n 49,\n 55,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_608d6493-6f0c-4ee6-bf03-c5c0e466c178.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:30:00.269756824Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::5]:48330","req_id":"7cec0089-5488-4466-8d9c-4f0129f5a7fa","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46379,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_7e22d17e-6cb5-4ec2-ab53-2a2d5da86032.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_7e22d17e-6cb5-4ec2-ab53-2a2d5da86032.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 55,\n 101,\n 50,\n 50,\n 100,\n 49,\n 55,\n 101,\n 45,\n 54,\n 99,\n 98,\n 53,\n 45,\n 52,\n 101,\n 99,\n 50,\n 45,\n 97,\n 98,\n 53,\n 51,\n 45,\n 50,\n 97,\n 50,\n 100,\n 53,\n 100,\n 97,\n 56,\n 54,\n 48,\n 51,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_7e22d17e-6cb5-4ec2-ab53-2a2d5da86032.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:30:00.287659074Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::3]:55203","req_id":"9997e694-115d-4c3e-8447-71940acc1f1b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46379,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_7e22d17e-6cb5-4ec2-ab53-2a2d5da86032.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_7e22d17e-6cb5-4ec2-ab53-2a2d5da86032.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 55,\n 101,\n 50,\n 50,\n 100,\n 49,\n 55,\n 101,\n 45,\n 54,\n 99,\n 98,\n 53,\n 45,\n 52,\n 101,\n 99,\n 50,\n 45,\n 97,\n 98,\n 53,\n 51,\n 45,\n 50,\n 97,\n 50,\n 100,\n 53,\n 100,\n 97,\n 56,\n 54,\n 48,\n 51,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_7e22d17e-6cb5-4ec2-ab53-2a2d5da86032.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:30:00.288146229Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::3]:58234","req_id":"f93a4436-94be-4342-b1fd-49d645d8cd80","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23713,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_bc7c08f5-0b98-4523-a806-55443475923d.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_bc7c08f5-0b98-4523-a806-55443475923d.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 99,\n 55,\n 99,\n 48,\n 56,\n 102,\n 53,\n 45,\n 48,\n 98,\n 57,\n 56,\n 45,\n 52,\n 53,\n 50,\n 51,\n 45,\n 97,\n 56,\n 48,\n 54,\n 45,\n 53,\n 53,\n 52,\n 52,\n 51,\n 52,\n 55,\n 53,\n 57,\n 50,\n 51,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_bc7c08f5-0b98-4523-a806-55443475923d.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:30:00.314600772Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::6]:51506","req_id":"6272e44d-8c38-4fc6-a8ad-386cdbb9cc84","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23713,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_bc7c08f5-0b98-4523-a806-55443475923d.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_bc7c08f5-0b98-4523-a806-55443475923d.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 99,\n 55,\n 99,\n 48,\n 56,\n 102,\n 53,\n 45,\n 48,\n 98,\n 57,\n 56,\n 45,\n 52,\n 53,\n 50,\n 51,\n 45,\n 97,\n 56,\n 48,\n 54,\n 45,\n 53,\n 53,\n 52,\n 52,\n 51,\n 52,\n 55,\n 53,\n 57,\n 50,\n 51,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_bc7c08f5-0b98-4523-a806-55443475923d.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:30:00.315107627Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::6]:58355","req_id":"0a9dbeef-8667-4df7-a1c1-89de646082d2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43620,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_cc2ac978-906a-4c5f-87ab-21841ee7f5cf.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_cc2ac978-906a-4c5f-87ab-21841ee7f5cf.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 99,\n 99,\n 50,\n 97,\n 99,\n 57,\n 55,\n 56,\n 45,\n 57,\n 48,\n 54,\n 97,\n 45,\n 52,\n 99,\n 53,\n 102,\n 45,\n 56,\n 55,\n 97,\n 98,\n 45,\n 50,\n 49,\n 56,\n 52,\n 49,\n 101,\n 101,\n 55,\n 102,\n 53,\n 99,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_cc2ac978-906a-4c5f-87ab-21841ee7f5cf.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T22:30:00.373434979Z","hostname":"oxz_internal_dns_9beb9a7f-83bf-4c9d-ae29-516675fe96c8","pid":5518,"peer_addr":"[fd00:1122:3344:101::1b]:45952","req_id":"6484e6f1-9913-4cd5-8e92-e6f1290185a1","component":"dns"}