[ Apr 17 13:57:47 Enabled. ] [ Apr 17 13:57:47 Rereading configuration. ] [ Apr 17 13:57:48 Rereading configuration. ] [ Apr 17 13:57:49 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:2::1]:5353 --dns-address [fd00:1122:3344:2::1]:53 &"). ] [ Apr 17 13:57:49 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:49.54016622Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"dns_config":"Config { bind_address: [fd00:1122:3344:2::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:49.540326262Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:49.558845873Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:49.558860103Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:49.558929444Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"component":"dns","local_address":"[fd00:1122:3344:2::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:49.559225037Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:50.927299246Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:2::2]:40037"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:50.928110715Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"new_generation":1,"req_id":"1d243d5d-8cf4-4e9c-b583-f5cffacaf66e","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:50.928126975Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:50.943565721Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:50.943585821Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"new_generation":1,"req_id":"1d243d5d-8cf4-4e9c-b583-f5cffacaf66e","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-04-17T13:57:50.943596711Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"uri":"/config","method":"PUT","req_id":"1d243d5d-8cf4-4e9c-b583-f5cffacaf66e","remote_addr":"[fd00:1122:3344:2::2]:40037","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":15870,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18268,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T13:58:01.928189003Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::1b]:38792","req_id":"26fa5c4c-3ccd-4010-abfa-84c3e069813e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52065,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T13:58:01.932817366Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::1b]:38584","req_id":"1164548f-1b38-44e1-afba-7f8f22ae8016","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 27052,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T13:58:01.936915942Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::1b]:57488","req_id":"665baf94-7f6a-44c1-9c22-0bce9ab4467b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 44689,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-17T13:59:54.712324161Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::d]:51303","req_id":"8c0833aa-5b0a-4b89-a441-8216768382d1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 36872,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 53,\n 97,\n 49,\n 52,\n 54,\n 53,\n 100,\n 55,\n 45,\n 55,\n 99,\n 48,\n 56,\n 45,\n 52,\n 57,\n 97,\n 55,\n 45,\n 97,\n 52,\n 53,\n 56,\n 45,\n 52,\n 53,\n 50,\n 102,\n 97,\n 102,\n 55,\n 50,\n 54,\n 102,\n 98,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T13:59:58.041834523Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::e]:52035","req_id":"a862b806-d968-40f7-bef3-5e78de22e53a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 36872,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 53,\n 97,\n 49,\n 52,\n 54,\n 53,\n 100,\n 55,\n 45,\n 55,\n 99,\n 48,\n 56,\n 45,\n 52,\n 57,\n 97,\n 55,\n 45,\n 97,\n 52,\n 53,\n 56,\n 45,\n 52,\n 53,\n 50,\n 102,\n 97,\n 102,\n 55,\n 50,\n 54,\n 102,\n 98,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T13:59:58.052530995Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::e]:61360","req_id":"ca712b37-ee0e-4cb1-9a94-370ab5d3dd7c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64315,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_4971314e-6d14-4ef9-8e5b-38fd7fef4bec.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_4971314e-6d14-4ef9-8e5b-38fd7fef4bec.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 52,\n 57,\n 55,\n 49,\n 51,\n 49,\n 52,\n 101,\n 45,\n 54,\n 100,\n 49,\n 52,\n 45,\n 52,\n 101,\n 102,\n 57,\n 45,\n 56,\n 101,\n 53,\n 98,\n 45,\n 51,\n 56,\n 102,\n 100,\n 55,\n 102,\n 101,\n 102,\n 52,\n 98,\n 101,\n 99,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_4971314e-6d14-4ef9-8e5b-38fd7fef4bec.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.274494847Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::6]:62800","req_id":"815dd6c9-11c5-4eca-ab38-946f1347f544","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64315,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_4971314e-6d14-4ef9-8e5b-38fd7fef4bec.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_4971314e-6d14-4ef9-8e5b-38fd7fef4bec.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 52,\n 57,\n 55,\n 49,\n 51,\n 49,\n 52,\n 101,\n 45,\n 54,\n 100,\n 49,\n 52,\n 45,\n 52,\n 101,\n 102,\n 57,\n 45,\n 56,\n 101,\n 53,\n 98,\n 45,\n 51,\n 56,\n 102,\n 100,\n 55,\n 102,\n 101,\n 102,\n 52,\n 98,\n 101,\n 99,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_4971314e-6d14-4ef9-8e5b-38fd7fef4bec.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.27572137Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::6]:48467","req_id":"ace017fb-a98b-469c-8ca8-60b8a682d6ea","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53974,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_d16d24ac-1e09-4343-a9ff-dd00e7d0397e.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_d16d24ac-1e09-4343-a9ff-dd00e7d0397e.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 100,\n 49,\n 54,\n 100,\n 50,\n 52,\n 97,\n 99,\n 45,\n 49,\n 101,\n 48,\n 57,\n 45,\n 52,\n 51,\n 52,\n 51,\n 45,\n 97,\n 57,\n 102,\n 102,\n 45,\n 100,\n 100,\n 48,\n 48,\n 101,\n 55,\n 100,\n 48,\n 51,\n 57,\n 55,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_d16d24ac-1e09-4343-a9ff-dd00e7d0397e.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.450263538Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::7]:44238","req_id":"26ed4439-7386-4b4b-8314-650273c6b408","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53974,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_d16d24ac-1e09-4343-a9ff-dd00e7d0397e.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_d16d24ac-1e09-4343-a9ff-dd00e7d0397e.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 100,\n 49,\n 54,\n 100,\n 50,\n 52,\n 97,\n 99,\n 45,\n 49,\n 101,\n 48,\n 57,\n 45,\n 52,\n 51,\n 52,\n 51,\n 45,\n 97,\n 57,\n 102,\n 102,\n 45,\n 100,\n 100,\n 48,\n 48,\n 101,\n 55,\n 100,\n 48,\n 51,\n 57,\n 55,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_d16d24ac-1e09-4343-a9ff-dd00e7d0397e.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.450893335Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::7]:54400","req_id":"41986195-27e9-43dc-8e14-e5b2d018507b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59830,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_c081ecaf-4bc0-4d9c-a11b-ada86e0ef4b6.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_c081ecaf-4bc0-4d9c-a11b-ada86e0ef4b6.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 99,\n 48,\n 56,\n 49,\n 101,\n 99,\n 97,\n 102,\n 45,\n 52,\n 98,\n 99,\n 48,\n 45,\n 52,\n 100,\n 57,\n 99,\n 45,\n 97,\n 49,\n 49,\n 98,\n 45,\n 97,\n 100,\n 97,\n 56,\n 54,\n 101,\n 48,\n 101,\n 102,\n 52,\n 98,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_c081ecaf-4bc0-4d9c-a11b-ada86e0ef4b6.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.536066135Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::5]:39639","req_id":"fd80e573-1c36-4d7c-a7b5-69d189fcd1ea","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59830,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_c081ecaf-4bc0-4d9c-a11b-ada86e0ef4b6.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_c081ecaf-4bc0-4d9c-a11b-ada86e0ef4b6.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 99,\n 48,\n 56,\n 49,\n 101,\n 99,\n 97,\n 102,\n 45,\n 52,\n 98,\n 99,\n 48,\n 45,\n 52,\n 100,\n 57,\n 99,\n 45,\n 97,\n 49,\n 49,\n 98,\n 45,\n 97,\n 100,\n 97,\n 56,\n 54,\n 101,\n 48,\n 101,\n 102,\n 52,\n 98,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_c081ecaf-4bc0-4d9c-a11b-ada86e0ef4b6.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.538928907Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::5]:55998","req_id":"a9558103-6320-40f4-85bd-d2616c9abab1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31472,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_e4b74fd0-067a-46cd-a7ac-c531df1f7dfa.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_e4b74fd0-067a-46cd-a7ac-c531df1f7dfa.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 101,\n 52,\n 98,\n 55,\n 52,\n 102,\n 100,\n 48,\n 45,\n 48,\n 54,\n 55,\n 97,\n 45,\n 52,\n 54,\n 99,\n 100,\n 45,\n 97,\n 55,\n 97,\n 99,\n 45,\n 99,\n 53,\n 51,\n 49,\n 100,\n 102,\n 49,\n 102,\n 55,\n 100,\n 102,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_e4b74fd0-067a-46cd-a7ac-c531df1f7dfa.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.543248017Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::3]:34848","req_id":"6822bca7-b5f6-446a-a295-2370c76777f9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31472,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_e4b74fd0-067a-46cd-a7ac-c531df1f7dfa.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_e4b74fd0-067a-46cd-a7ac-c531df1f7dfa.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 101,\n 52,\n 98,\n 55,\n 52,\n 102,\n 100,\n 48,\n 45,\n 48,\n 54,\n 55,\n 97,\n 45,\n 52,\n 54,\n 99,\n 100,\n 45,\n 97,\n 55,\n 97,\n 99,\n 45,\n 99,\n 53,\n 51,\n 49,\n 100,\n 102,\n 49,\n 102,\n 55,\n 100,\n 102,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_e4b74fd0-067a-46cd-a7ac-c531df1f7dfa.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.543811293Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::3]:49361","req_id":"c00d1aef-1075-42a6-b2f1-fa9539cad552","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 35542,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_26dc7d7d-b8f6-4855-a7b9-60481b3c37af.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_26dc7d7d-b8f6-4855-a7b9-60481b3c37af.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 50,\n 54,\n 100,\n 99,\n 55,\n 100,\n 55,\n 100,\n 45,\n 98,\n 56,\n 102,\n 54,\n 45,\n 52,\n 56,\n 53,\n 53,\n 45,\n 97,\n 55,\n 98,\n 57,\n 45,\n 54,\n 48,\n 52,\n 56,\n 49,\n 98,\n 51,\n 99,\n 51,\n 55,\n 97,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_26dc7d7d-b8f6-4855-a7b9-60481b3c37af.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.562469026Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::4]:35185","req_id":"9f882c9a-b79d-4d2c-9164-fea6bce17da1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 35542,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_26dc7d7d-b8f6-4855-a7b9-60481b3c37af.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_26dc7d7d-b8f6-4855-a7b9-60481b3c37af.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 50,\n 54,\n 100,\n 99,\n 55,\n 100,\n 55,\n 100,\n 45,\n 98,\n 56,\n 102,\n 54,\n 45,\n 52,\n 56,\n 53,\n 53,\n 45,\n 97,\n 55,\n 98,\n 57,\n 45,\n 54,\n 48,\n 52,\n 56,\n 49,\n 98,\n 51,\n 99,\n 51,\n 55,\n 97,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_26dc7d7d-b8f6-4855-a7b9-60481b3c37af.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.563246754Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::4]:60385","req_id":"955edb68-a5a1-40cd-9c42-4773e1826bc0","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 124,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_0ed75755-7f49-4067-8976-5cc52b736946.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_0ed75755-7f49-4067-8976-5cc52b736946.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 48,\n 101,\n 100,\n 55,\n 53,\n 55,\n 53,\n 53,\n 45,\n 55,\n 102,\n 52,\n 57,\n 45,\n 52,\n 48,\n 54,\n 55,\n 45,\n 56,\n 57,\n 55,\n 54,\n 45,\n 53,\n 99,\n 99,\n 53,\n 50,\n 98,\n 55,\n 51,\n 54,\n 57,\n 52,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_0ed75755-7f49-4067-8976-5cc52b736946.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.665224886Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::1b]:36385","req_id":"65d89be2-e9b6-4684-829a-28a77b4aaa00","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 2636,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 53,\n 97,\n 49,\n 52,\n 54,\n 53,\n 100,\n 55,\n 45,\n 55,\n 99,\n 48,\n 56,\n 45,\n 52,\n 57,\n 97,\n 55,\n 45,\n 97,\n 52,\n 53,\n 56,\n 45,\n 52,\n 53,\n 50,\n 102,\n 97,\n 102,\n 55,\n 50,\n 54,\n 102,\n 98,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.703420541Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::e]:49530","req_id":"5c17fef1-35f4-46f8-8989-955a02c61bc6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 2636,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 53,\n 97,\n 49,\n 52,\n 54,\n 53,\n 100,\n 55,\n 45,\n 55,\n 99,\n 48,\n 56,\n 45,\n 52,\n 57,\n 97,\n 55,\n 45,\n 97,\n 52,\n 53,\n 56,\n 45,\n 52,\n 53,\n 50,\n 102,\n 97,\n 102,\n 55,\n 50,\n 54,\n 102,\n 98,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_5a1465d7-7c08-49a7-a458-452faf726fba.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:00.70423581Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::e]:48434","req_id":"3fda28fb-50cb-4f03-9f15-ad66a1aa49ec","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34012,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:01.410679625Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::2]:39352","req_id":"5e1be772-55b2-45fc-ac6c-aacf940fca32","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34012,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-17T14:00:01.411300332Z","hostname":"oxz_internal_dns_a09a8540-7084-4bfd-9942-1eabb07dede6","pid":5496,"peer_addr":"[fd00:1122:3344:101::2]:39626","req_id":"a2080c04-a8e0-4f78-85b8-87d629ea09cf","component":"dns"}