[ Jul 16 18:56:40 Enabled. ] [ Jul 16 18:56:40 Rereading configuration. ] [ Jul 16 18:56:40 Rereading configuration. ] [ Jul 16 18:56:41 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:2::1]:5353 --dns-address [fd00:1122:3344:2::1]:53 &"). ] [ Jul 16 18:56:41 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-07-16T18:56:41.666031531Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"dns_config":"Config { bind_address: [fd00:1122:3344:2::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-07-16T18:56:41.666169224Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-07-16T18:56:41.687427721Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-07-16T18:56:41.687461602Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-07-16T18:56:41.687491632Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"component":"dns","local_address":"[fd00:1122:3344:2::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-07-16T18:56:41.687679526Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3682,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-16T18:56:41.765085512Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::2]:53932","req_id":"0188cdbf-bd74-4b32-99a1-fa960f0efd3e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 11822,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-16T18:56:41.894051264Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::2]:17613","req_id":"f8f17db4-8ce8-4d0a-abe1-1ad33bec1078","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 5068,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-16T18:56:41.99291049Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::2]:38291","req_id":"8782a31a-054c-4bd6-b8e9-a6e1ef972493","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 8944,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-16T18:56:42.008475769Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::2]:40969","req_id":"531ab917-0521-4d3e-8413-2b8b85e05f1b","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-07-16T18:56:42.11287638Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:2::2]:40421"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-07-16T18:56:42.113538005Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"new_generation":1,"req_id":"a91e4ca3-adfa-4454-819b-e576037f02db","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-07-16T18:56:42.113584176Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-07-16T18:56:42.125831711Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-07-16T18:56:42.125863232Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"new_generation":1,"req_id":"a91e4ca3-adfa-4454-819b-e576037f02db","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-07-16T18:56:42.125872052Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"uri":"/config","method":"PUT","req_id":"a91e4ca3-adfa-4454-819b-e576037f02db","remote_addr":"[fd00:1122:3344:2::2]:40421","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":12667,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1409,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-16T18:56:52.951321034Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::1b]:65023","req_id":"a1605dd9-3680-4a0f-a17c-4e88c25b025d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 19677,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-16T18:56:52.955918827Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::1b]:45566","req_id":"a835a382-abbe-4506-8670-723c7955dea4","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34752,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-16T18:56:52.959756844Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::1b]:61027","req_id":"d7bba220-c423-483f-8d12-36dd3f480e23","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 39452,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-07-16T18:59:57.873456858Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::d]:59501","req_id":"c318b62b-1cc2-44f3-9261-754045aecf4a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 29953,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_95b0620a-77fd-429e-a5cb-00a5e779a0fb.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_95b0620a-77fd-429e-a5cb-00a5e779a0fb.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 57,\n 53,\n 98,\n 48,\n 54,\n 50,\n 48,\n 97,\n 45,\n 55,\n 55,\n 102,\n 100,\n 45,\n 52,\n 50,\n 57,\n 101,\n 45,\n 97,\n 53,\n 99,\n 98,\n 45,\n 48,\n 48,\n 97,\n 53,\n 101,\n 55,\n 55,\n 57,\n 97,\n 48,\n 102,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_95b0620a-77fd-429e-a5cb-00a5e779a0fb.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-16T19:00:00.301560095Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::7]:41201","req_id":"ddd94e5b-a328-48ed-89bc-8c16a2dd7c3a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 61974,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_710f6d32-efb7-452e-948f-6330a258a19b.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_710f6d32-efb7-452e-948f-6330a258a19b.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 55,\n 49,\n 48,\n 102,\n 54,\n 100,\n 51,\n 50,\n 45,\n 101,\n 102,\n 98,\n 55,\n 45,\n 52,\n 53,\n 50,\n 101,\n 45,\n 57,\n 52,\n 56,\n 102,\n 45,\n 54,\n 51,\n 51,\n 48,\n 97,\n 50,\n 53,\n 56,\n 97,\n 49,\n 57,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_710f6d32-efb7-452e-948f-6330a258a19b.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-16T19:00:00.305619476Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::4]:55762","req_id":"20ed0016-d9c6-4372-b130-f312dd4c6e8b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 61974,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_710f6d32-efb7-452e-948f-6330a258a19b.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_710f6d32-efb7-452e-948f-6330a258a19b.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 55,\n 49,\n 48,\n 102,\n 54,\n 100,\n 51,\n 50,\n 45,\n 101,\n 102,\n 98,\n 55,\n 45,\n 52,\n 53,\n 50,\n 101,\n 45,\n 57,\n 52,\n 56,\n 102,\n 45,\n 54,\n 51,\n 51,\n 48,\n 97,\n 50,\n 53,\n 56,\n 97,\n 49,\n 57,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_710f6d32-efb7-452e-948f-6330a258a19b.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-16T19:00:00.307290893Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::4]:32802","req_id":"70320f41-4462-44cc-a078-e317e0ded84c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 29953,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_95b0620a-77fd-429e-a5cb-00a5e779a0fb.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_95b0620a-77fd-429e-a5cb-00a5e779a0fb.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 57,\n 53,\n 98,\n 48,\n 54,\n 50,\n 48,\n 97,\n 45,\n 55,\n 55,\n 102,\n 100,\n 45,\n 52,\n 50,\n 57,\n 101,\n 45,\n 97,\n 53,\n 99,\n 98,\n 45,\n 48,\n 48,\n 97,\n 53,\n 101,\n 55,\n 55,\n 57,\n 97,\n 48,\n 102,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_95b0620a-77fd-429e-a5cb-00a5e779a0fb.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-16T19:00:00.307338035Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::7]:47277","req_id":"3f0066e0-4e55-4d85-a0b8-b9b72dd41f23","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42578,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_0a445da3-17ce-4a42-9268-57fbdd124f2e.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_0a445da3-17ce-4a42-9268-57fbdd124f2e.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 48,\n 97,\n 52,\n 52,\n 53,\n 100,\n 97,\n 51,\n 45,\n 49,\n 55,\n 99,\n 101,\n 45,\n 52,\n 97,\n 52,\n 50,\n 45,\n 57,\n 50,\n 54,\n 56,\n 45,\n 53,\n 55,\n 102,\n 98,\n 100,\n 100,\n 49,\n 50,\n 52,\n 102,\n 50,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_0a445da3-17ce-4a42-9268-57fbdd124f2e.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-16T19:00:00.333895129Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::5]:44284","req_id":"345a3c4f-5cda-4933-9f26-1fdcea6798df","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42578,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_0a445da3-17ce-4a42-9268-57fbdd124f2e.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_0a445da3-17ce-4a42-9268-57fbdd124f2e.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 48,\n 97,\n 52,\n 52,\n 53,\n 100,\n 97,\n 51,\n 45,\n 49,\n 55,\n 99,\n 101,\n 45,\n 52,\n 97,\n 52,\n 50,\n 45,\n 57,\n 50,\n 54,\n 56,\n 45,\n 53,\n 55,\n 102,\n 98,\n 100,\n 100,\n 49,\n 50,\n 52,\n 102,\n 50,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_0a445da3-17ce-4a42-9268-57fbdd124f2e.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-16T19:00:00.334511033Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::5]:50421","req_id":"2348dc83-0984-40c1-a9be-45f57d445eef","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 6214,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_80c09338-e228-43e6-a0c6-5ddafc670103.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_80c09338-e228-43e6-a0c6-5ddafc670103.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 48,\n 99,\n 48,\n 57,\n 51,\n 51,\n 56,\n 45,\n 101,\n 50,\n 50,\n 56,\n 45,\n 52,\n 51,\n 101,\n 54,\n 45,\n 97,\n 48,\n 99,\n 54,\n 45,\n 53,\n 100,\n 100,\n 97,\n 102,\n 99,\n 54,\n 55,\n 48,\n 49,\n 48,\n 51,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_80c09338-e228-43e6-a0c6-5ddafc670103.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-16T19:00:00.342929652Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::6]:56472","req_id":"6fcf077a-46ea-4102-80ee-ddf46fa77020","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 6214,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_80c09338-e228-43e6-a0c6-5ddafc670103.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_80c09338-e228-43e6-a0c6-5ddafc670103.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 56,\n 48,\n 99,\n 48,\n 57,\n 51,\n 51,\n 56,\n 45,\n 101,\n 50,\n 50,\n 56,\n 45,\n 52,\n 51,\n 101,\n 54,\n 45,\n 97,\n 48,\n 99,\n 54,\n 45,\n 53,\n 100,\n 100,\n 97,\n 102,\n 99,\n 54,\n 55,\n 48,\n 49,\n 48,\n 51,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_80c09338-e228-43e6-a0c6-5ddafc670103.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-16T19:00:00.343543255Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::6]:46225","req_id":"a123f9c0-d3f4-4d7d-8759-32d390f8c9a5","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 44172,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_ca92e564-dc81-4c05-a583-a6e9b1add56a.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_ca92e564-dc81-4c05-a583-a6e9b1add56a.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 99,\n 97,\n 57,\n 50,\n 101,\n 53,\n 54,\n 52,\n 45,\n 100,\n 99,\n 56,\n 49,\n 45,\n 52,\n 99,\n 48,\n 53,\n 45,\n 97,\n 53,\n 56,\n 51,\n 45,\n 97,\n 54,\n 101,\n 57,\n 98,\n 49,\n 97,\n 100,\n 100,\n 53,\n 54,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_ca92e564-dc81-4c05-a583-a6e9b1add56a.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-16T19:00:00.378469599Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::3]:56446","req_id":"09db116e-50a1-4b40-b90d-f2583406fcea","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 44172,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_ca92e564-dc81-4c05-a583-a6e9b1add56a.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_ca92e564-dc81-4c05-a583-a6e9b1add56a.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 99,\n 97,\n 57,\n 50,\n 101,\n 53,\n 54,\n 52,\n 45,\n 100,\n 99,\n 56,\n 49,\n 45,\n 52,\n 99,\n 48,\n 53,\n 45,\n 97,\n 53,\n 56,\n 51,\n 45,\n 97,\n 54,\n 101,\n 57,\n 98,\n 49,\n 97,\n 100,\n 100,\n 53,\n 54,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_ca92e564-dc81-4c05-a583-a6e9b1add56a.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-16T19:00:00.379213125Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::3]:54245","req_id":"cf5ef33a-a7fe-4d0d-837d-df86c9028193","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 24858,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_27231902-fe6c-4510-820a-3d920d3c8989.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_27231902-fe6c-4510-820a-3d920d3c8989.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 50,\n 55,\n 50,\n 51,\n 49,\n 57,\n 48,\n 50,\n 45,\n 102,\n 101,\n 54,\n 99,\n 45,\n 52,\n 53,\n 49,\n 48,\n 45,\n 56,\n 50,\n 48,\n 97,\n 45,\n 51,\n 100,\n 57,\n 50,\n 48,\n 100,\n 51,\n 99,\n 56,\n 57,\n 56,\n 57,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_27231902-fe6c-4510-820a-3d920d3c8989.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-16T19:00:00.639678426Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::1b]:34303","req_id":"bda7910e-8ac6-4d1d-9f5e-83e75b2c84ce","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42457,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-16T19:00:01.13041812Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::2]:56247","req_id":"b1705735-3464-4576-a568-0dd9077db236","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 42457,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-07-16T19:00:01.131061825Z","hostname":"oxz_internal_dns_3e820383-2a0d-4e4a-af31-3b1a0aa2588b","pid":5410,"peer_addr":"[fd00:1122:3344:101::2]:61454","req_id":"48f5ebfa-4eba-4ff2-a28a-b0b982530db1","component":"dns"}