[ Aug 20 04:43:14 Enabled. ] [ Aug 20 04:43:14 Rereading configuration. ] [ Aug 20 04:43:15 Rereading configuration. ] [ Aug 20 04:43:16 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:1::1]:5353 --dns-address [fd00:1122:3344:1::1]:53 &"). ] [ Aug 20 04:43:16 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:1::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-08-20T04:43:16.249477759Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"dns_config":"Config { bind_address: [fd00:1122:3344:1::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:1::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-08-20T04:43:16.249653442Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-08-20T04:43:16.270542926Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-08-20T04:43:16.270565606Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-08-20T04:43:16.270588437Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"component":"dns","local_address":"[fd00:1122:3344:1::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-08-20T04:43:16.27081045Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 5403,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:43:16.290640366Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::2]:35009","req_id":"3d59f888-8229-4f0c-be91-23d3fc6d5873","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3018,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:43:16.409428648Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::2]:9360","req_id":"fcc3e720-f2cf-4fa7-817f-e69768845e6f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64309,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:43:16.548418462Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::2]:48912","req_id":"aef5fb08-1701-4cea-8e7e-c2dc0fcbd5dc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 19399,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:43:16.561022695Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::2]:28067","req_id":"f3b0f94c-933f-482a-ac7a-030b68507bc7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30614,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:43:16.584157827Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::2]:58911","req_id":"b718bb87-feca-4263-a8cf-bb8b37faa4eb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 55475,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:43:16.76382757Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::2]:20849","req_id":"e19de4ed-8f36-425d-b180-c19ca6bd18ab","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-08-20T04:43:16.825555065Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:1::2]:46322"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-08-20T04:43:16.826367689Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"new_generation":1,"req_id":"c31580df-e99c-431b-8af1-e0811904c307","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-08-20T04:43:16.826397139Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"component":"store"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16170,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:43:16.835158948Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::2]:4140","req_id":"cfe36f40-cac0-4166-92ea-b17d219ed108","component":"dns"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-08-20T04:43:16.842821708Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-08-20T04:43:16.842845858Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"new_generation":1,"req_id":"c31580df-e99c-431b-8af1-e0811904c307","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-08-20T04:43:16.842863528Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"uri":"/config","method":"PUT","req_id":"c31580df-e99c-431b-8af1-e0811904c307","remote_addr":"[fd00:1122:3344:1::2]:46322","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":16904,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18150,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:43:28.092613093Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::1b]:38395","req_id":"b3993da7-9518-4c33-a8d8-d54ed221a9a7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 10111,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:43:28.097515526Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::1b]:38294","req_id":"02ed37cd-1d15-4efb-a8eb-a1e2ac421b94","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 63969,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:43:28.10187794Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::1b]:49357","req_id":"647b2f63-9c51-46d4-8efe-8b22228d751a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21953,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:44:41.270008497Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::d]:63637","req_id":"56de802f-4050-4a56-a7a9-f6650226462e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54403,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_db06bc30-417a-4358-b8a7-9ad304eb1b48.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_db06bc30-417a-4358-b8a7-9ad304eb1b48.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 100,\n 98,\n 48,\n 54,\n 98,\n 99,\n 51,\n 48,\n 45,\n 52,\n 49,\n 55,\n 97,\n 45,\n 52,\n 51,\n 53,\n 56,\n 45,\n 98,\n 56,\n 97,\n 55,\n 45,\n 57,\n 97,\n 100,\n 51,\n 48,\n 52,\n 101,\n 98,\n 49,\n 98,\n 52,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_db06bc30-417a-4358-b8a7-9ad304eb1b48.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:44:47.847610667Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::e]:50615","req_id":"3486dc65-ba45-4a04-b6ea-d92bc4733b87","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54403,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_db06bc30-417a-4358-b8a7-9ad304eb1b48.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_db06bc30-417a-4358-b8a7-9ad304eb1b48.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 100,\n 98,\n 48,\n 54,\n 98,\n 99,\n 51,\n 48,\n 45,\n 52,\n 49,\n 55,\n 97,\n 45,\n 52,\n 51,\n 53,\n 56,\n 45,\n 98,\n 56,\n 97,\n 55,\n 45,\n 57,\n 97,\n 100,\n 51,\n 48,\n 52,\n 101,\n 98,\n 49,\n 98,\n 52,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_db06bc30-417a-4358-b8a7-9ad304eb1b48.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:44:47.868378199Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::e]:63940","req_id":"0bc62c7c-7969-4a81-b811-8f2cfe6d6321","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3152,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:44:51.256125001Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::d]:53568","req_id":"b03b05ce-25fc-4703-a298-27b308d2cc4a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42097,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:44:52.744194502Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::b]:62896","req_id":"064583f4-845b-4808-8cd8-026c4a338658","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 56648,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:44:52.982206683Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::c]:4500","req_id":"30db57a4-d2f0-4433-a40e-909e57463e38","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 44725,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:44:53.148276105Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::a]:9819","req_id":"e5dba2b5-a43d-4a3a-8d63-cd6d2c9abbea","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 45527,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_d3c66b4a-aa3f-4322-960e-0bdc4fb3e0fa.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_d3c66b4a-aa3f-4322-960e-0bdc4fb3e0fa.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 100,\n 51,\n 99,\n 54,\n 54,\n 98,\n 52,\n 97,\n 45,\n 97,\n 97,\n 51,\n 102,\n 45,\n 52,\n 51,\n 50,\n 50,\n 45,\n 57,\n 54,\n 48,\n 101,\n 45,\n 48,\n 98,\n 100,\n 99,\n 52,\n 102,\n 98,\n 51,\n 101,\n 48,\n 102,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_d3c66b4a-aa3f-4322-960e-0bdc4fb3e0fa.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:45:00.116264498Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::1b]:42555","req_id":"74aa8513-8840-4035-a27d-a713a97b0fbc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 9893,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:45:00.262181869Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::2]:63840","req_id":"f1a0677a-15e2-4429-b2e6-7d03e19599eb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 9893,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:45:00.26288198Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::2]:41944","req_id":"4c8310b8-2a04-40bb-9269-3ddc13d5fd65","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 715,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_db06bc30-417a-4358-b8a7-9ad304eb1b48.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_db06bc30-417a-4358-b8a7-9ad304eb1b48.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 100,\n 98,\n 48,\n 54,\n 98,\n 99,\n 51,\n 48,\n 45,\n 52,\n 49,\n 55,\n 97,\n 45,\n 52,\n 51,\n 53,\n 56,\n 45,\n 98,\n 56,\n 97,\n 55,\n 45,\n 57,\n 97,\n 100,\n 51,\n 48,\n 52,\n 101,\n 98,\n 49,\n 98,\n 52,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_db06bc30-417a-4358-b8a7-9ad304eb1b48.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:45:00.437701201Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::e]:38182","req_id":"8bfc629a-bfb0-4a68-9ad2-a1f5193af111","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 715,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_db06bc30-417a-4358-b8a7-9ad304eb1b48.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_db06bc30-417a-4358-b8a7-9ad304eb1b48.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 100,\n 98,\n 48,\n 54,\n 98,\n 99,\n 51,\n 48,\n 45,\n 52,\n 49,\n 55,\n 97,\n 45,\n 52,\n 51,\n 53,\n 56,\n 45,\n 98,\n 56,\n 97,\n 55,\n 45,\n 57,\n 97,\n 100,\n 51,\n 48,\n 52,\n 101,\n 98,\n 49,\n 98,\n 52,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_db06bc30-417a-4358-b8a7-9ad304eb1b48.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:45:00.438371922Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::e]:40177","req_id":"55162d9b-56fb-4ff9-ae51-fa2d3d5345a0","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38873,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_8390548b-0fed-404e-81c7-081e288f1bc7.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_8390548b-0fed-404e-81c7-081e288f1bc7.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 51,\n 57,\n 48,\n 53,\n 52,\n 56,\n 98,\n 45,\n 48,\n 102,\n 101,\n 100,\n 45,\n 52,\n 48,\n 52,\n 101,\n 45,\n 56,\n 49,\n 99,\n 55,\n 45,\n 48,\n 56,\n 49,\n 101,\n 50,\n 56,\n 56,\n 102,\n 49,\n 98,\n 99,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_8390548b-0fed-404e-81c7-081e288f1bc7.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:45:00.61949088Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::4]:56591","req_id":"f9195e70-167b-4e5e-8fbb-79cedc0d23b1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38873,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_8390548b-0fed-404e-81c7-081e288f1bc7.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_8390548b-0fed-404e-81c7-081e288f1bc7.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 51,\n 57,\n 48,\n 53,\n 52,\n 56,\n 98,\n 45,\n 48,\n 102,\n 101,\n 100,\n 45,\n 52,\n 48,\n 52,\n 101,\n 45,\n 56,\n 49,\n 99,\n 55,\n 45,\n 48,\n 56,\n 49,\n 101,\n 50,\n 56,\n 56,\n 102,\n 49,\n 98,\n 99,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_8390548b-0fed-404e-81c7-081e288f1bc7.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:45:00.620218622Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::4]:48131","req_id":"731a8302-1f89-4b2e-b5a6-b130d46109fb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42364,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_3cd6d523-6ae7-43ca-959f-81fadbb3bd9b.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_3cd6d523-6ae7-43ca-959f-81fadbb3bd9b.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 99,\n 100,\n 54,\n 100,\n 53,\n 50,\n 51,\n 45,\n 54,\n 97,\n 101,\n 55,\n 45,\n 52,\n 51,\n 99,\n 97,\n 45,\n 57,\n 53,\n 57,\n 102,\n 45,\n 56,\n 49,\n 102,\n 97,\n 100,\n 98,\n 98,\n 51,\n 98,\n 100,\n 57,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_3cd6d523-6ae7-43ca-959f-81fadbb3bd9b.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:45:00.639789933Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::5]:36502","req_id":"a26329b4-3db8-42e6-9c25-83e27c2cbacf","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42364,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_3cd6d523-6ae7-43ca-959f-81fadbb3bd9b.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_3cd6d523-6ae7-43ca-959f-81fadbb3bd9b.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 99,\n 100,\n 54,\n 100,\n 53,\n 50,\n 51,\n 45,\n 54,\n 97,\n 101,\n 55,\n 45,\n 52,\n 51,\n 99,\n 97,\n 45,\n 57,\n 53,\n 57,\n 102,\n 45,\n 56,\n 49,\n 102,\n 97,\n 100,\n 98,\n 98,\n 51,\n 98,\n 100,\n 57,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_3cd6d523-6ae7-43ca-959f-81fadbb3bd9b.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:45:00.640618289Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::5]:63452","req_id":"31904775-3b36-419d-a5ba-3bcb33838147","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 27252,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_3f833d36-b26d-49a7-8248-7e14586dbba4.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_3f833d36-b26d-49a7-8248-7e14586dbba4.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 102,\n 56,\n 51,\n 51,\n 100,\n 51,\n 54,\n 45,\n 98,\n 50,\n 54,\n 100,\n 45,\n 52,\n 57,\n 97,\n 55,\n 45,\n 56,\n 50,\n 52,\n 56,\n 45,\n 55,\n 101,\n 49,\n 52,\n 53,\n 56,\n 54,\n 100,\n 98,\n 98,\n 97,\n 52,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_3f833d36-b26d-49a7-8248-7e14586dbba4.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:45:00.673338511Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::7]:63222","req_id":"63b1fe3f-c2f0-4d8c-bc97-d7587ef17ca6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 27252,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_3f833d36-b26d-49a7-8248-7e14586dbba4.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_3f833d36-b26d-49a7-8248-7e14586dbba4.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 102,\n 56,\n 51,\n 51,\n 100,\n 51,\n 54,\n 45,\n 98,\n 50,\n 54,\n 100,\n 45,\n 52,\n 57,\n 97,\n 55,\n 45,\n 56,\n 50,\n 52,\n 56,\n 45,\n 55,\n 101,\n 49,\n 52,\n 53,\n 56,\n 54,\n 100,\n 98,\n 98,\n 97,\n 52,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_3f833d36-b26d-49a7-8248-7e14586dbba4.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:45:00.673991663Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::7]:38542","req_id":"d2d8973d-a6da-45af-a874-5e64ffb97a39","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16743,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_465304db-09bd-4e61-a1ae-9044f369936f.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_465304db-09bd-4e61-a1ae-9044f369936f.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 52,\n 54,\n 53,\n 51,\n 48,\n 52,\n 100,\n 98,\n 45,\n 48,\n 57,\n 98,\n 100,\n 45,\n 52,\n 101,\n 54,\n 49,\n 45,\n 97,\n 49,\n 97,\n 101,\n 45,\n 57,\n 48,\n 52,\n 52,\n 102,\n 51,\n 54,\n 57,\n 57,\n 51,\n 54,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_465304db-09bd-4e61-a1ae-9044f369936f.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:45:00.692994594Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::6]:63572","req_id":"51343415-d116-4198-ac77-a6320d433cbd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16743,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_465304db-09bd-4e61-a1ae-9044f369936f.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_465304db-09bd-4e61-a1ae-9044f369936f.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 52,\n 54,\n 53,\n 51,\n 48,\n 52,\n 100,\n 98,\n 45,\n 48,\n 57,\n 98,\n 100,\n 45,\n 52,\n 101,\n 54,\n 49,\n 45,\n 97,\n 49,\n 97,\n 101,\n 45,\n 57,\n 48,\n 52,\n 52,\n 102,\n 51,\n 54,\n 57,\n 57,\n 51,\n 54,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_465304db-09bd-4e61-a1ae-9044f369936f.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:45:00.693802058Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::6]:51212","req_id":"fc1b8de0-0e33-4552-a768-b4165cc9eb37","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31606,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_923e8477-c46d-4b84-b932-14195b7f5053.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_923e8477-c46d-4b84-b932-14195b7f5053.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 57,\n 50,\n 51,\n 101,\n 56,\n 52,\n 55,\n 55,\n 45,\n 99,\n 52,\n 54,\n 100,\n 45,\n 52,\n 98,\n 56,\n 52,\n 45,\n 98,\n 57,\n 51,\n 50,\n 45,\n 49,\n 52,\n 49,\n 57,\n 53,\n 98,\n 55,\n 102,\n 53,\n 48,\n 53,\n 51,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_923e8477-c46d-4b84-b932-14195b7f5053.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:45:00.693998361Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::3]:58431","req_id":"6f2aca81-b57f-474c-98ba-745ad222bf6e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31606,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_923e8477-c46d-4b84-b932-14195b7f5053.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_923e8477-c46d-4b84-b932-14195b7f5053.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 57,\n 50,\n 51,\n 101,\n 56,\n 52,\n 55,\n 55,\n 45,\n 99,\n 52,\n 54,\n 100,\n 45,\n 52,\n 98,\n 56,\n 52,\n 45,\n 98,\n 57,\n 51,\n 50,\n 45,\n 49,\n 52,\n 49,\n 57,\n 53,\n 98,\n 55,\n 102,\n 53,\n 48,\n 53,\n 51,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_923e8477-c46d-4b84-b932-14195b7f5053.\"","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:45:00.694364168Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::3]:57318","req_id":"df4e190e-a7c4-45a7-90a7-4ef59360b8e4","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 25843,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-08-20T04:45:01.258084715Z","hostname":"oxz_internal_dns_51995292-e259-4829-9648-0476382220e7","pid":5426,"peer_addr":"[fd00:1122:3344:101::d]:58655","req_id":"b5ac1ee9-9d5c-40ed-b56f-423d20bf4ac3","component":"dns"}