[ Apr 10 03:10:08 Enabled. ] [ Apr 10 03:10:08 Rereading configuration. ] [ Apr 10 03:10:09 Rereading configuration. ] [ Apr 10 03:10:10 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:3::1]:5353 --dns-address [fd00:1122:3344:3::1]:53 &"). ] [ Apr 10 03:10:11 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:10:11.014696846Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"dns_config":"Config { bind_address: [fd00:1122:3344:3::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:10:11.014871439Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:10:11.033509969Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:10:11.03353273Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:10:11.033569651Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"component":"dns","local_address":"[fd00:1122:3344:3::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:10:11.033889577Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14724,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 6,\n 95,\n 110,\n 101,\n 120,\n 117,\n 115,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:10:11.345830461Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:3::2]:53221","req_id":"77b51a47-28eb-4885-b7de-4df16839b661","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:10:11.418744289Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:3::2]:59919"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:10:11.419385902Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"new_generation":1,"req_id":"71dcbc2b-aee2-4d58-afdb-4c42c1d72930","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:10:11.419399232Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:10:11.464789503Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:10:11.464820494Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"new_generation":1,"req_id":"71dcbc2b-aee2-4d58-afdb-4c42c1d72930","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:10:11.464837684Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"uri":"/config","method":"PUT","req_id":"71dcbc2b-aee2-4d58-afdb-4c42c1d72930","remote_addr":"[fd00:1122:3344:3::2]:59919","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":45772,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57235,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:10:23.762773804Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::1b]:36208","req_id":"01d2c9f2-dc6b-4afc-b0df-ce7843138010","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 17752,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:10:23.769299294Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::1b]:63367","req_id":"eb68d840-a7ff-4da2-b0dc-1dbeb4e540ca","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54598,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:10:23.774363825Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::1b]:41887","req_id":"3cfa0491-3c3f-49d6-b071-59c0ec58e673","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30478,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:11:48.403114496Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::1b]:56635","req_id":"10e6d850-4ecf-4f99-872c-eb2a5016af44","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 7615,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:11:48.408070965Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::1b]:55502","req_id":"f8dd0c73-bdbc-423a-ad0a-c3493b5ccf60","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43544,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:11:48.412596908Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::1b]:47630","req_id":"7ff7b7c0-b12f-4286-941a-9df4f516878c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 58444,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:13:44.597966653Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::d]:65488","req_id":"73d23f64-20fd-4cc7-ad46-6fd9ba7f4be6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43122,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_943626f6-bfbc-4742-9c48-17f14e613b68.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_943626f6-bfbc-4742-9c48-17f14e613b68.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 57,\n 52,\n 51,\n 54,\n 50,\n 54,\n 102,\n 54,\n 45,\n 98,\n 102,\n 98,\n 99,\n 45,\n 52,\n 55,\n 52,\n 50,\n 45,\n 57,\n 99,\n 52,\n 56,\n 45,\n 49,\n 55,\n 102,\n 49,\n 52,\n 101,\n 54,\n 49,\n 51,\n 98,\n 54,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_943626f6-bfbc-4742-9c48-17f14e613b68.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:13:47.305075478Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::e]:41002","req_id":"6990b2e3-68b7-465d-b08e-ea15e6973331","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43122,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_943626f6-bfbc-4742-9c48-17f14e613b68.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_943626f6-bfbc-4742-9c48-17f14e613b68.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 57,\n 52,\n 51,\n 54,\n 50,\n 54,\n 102,\n 54,\n 45,\n 98,\n 102,\n 98,\n 99,\n 45,\n 52,\n 55,\n 52,\n 50,\n 45,\n 57,\n 99,\n 52,\n 56,\n 45,\n 49,\n 55,\n 102,\n 49,\n 52,\n 101,\n 54,\n 49,\n 51,\n 98,\n 54,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_943626f6-bfbc-4742-9c48-17f14e613b68.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:13:47.30518001Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::e]:37260","req_id":"fd98bad7-c0ec-44c9-8644-8bbd7c3da534","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64293,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:13:52.129743462Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::a]:49552","req_id":"59c38f8e-b9fe-4e56-b3c7-6eb622d8174f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 11103,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:13:53.346948712Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::c]:40172","req_id":"3d354e06-7a5e-4161-b260-3f54779ead27","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 5488,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:13:53.535844553Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::b]:62759","req_id":"b99ac8a9-2225-497e-ac0f-0d8cc4d5f973","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:13:54.093850034Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:55000"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:13:54.09668649Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:33894"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:13:54.133387259Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"uri":"/config","method":"GET","req_id":"2285debe-36a0-47b7-a8dd-4b4c8c66d72d","remote_addr":"[fd00:1122:3344:101::c]:55000","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":26370,"response_code":200} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:13:54.13346432Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"uri":"/config","method":"GET","req_id":"a1f00bd5-1465-43c2-853c-9b5dfc1dadad","remote_addr":"[fd00:1122:3344:101::b]:33894","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":26371,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 58418,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:13:54.149895466Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::a]:35635","req_id":"ca2a3fcb-3573-4cfa-b4f9-2deeccfa5192","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:13:54.271812018Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:53268"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:13:54.273222446Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"uri":"/config","method":"GET","req_id":"44e4ed6c-f1b8-4339-b76b-89c67d3228cc","remote_addr":"[fd00:1122:3344:101::a]:53268","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":1349,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 28352,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:13:55.723408432Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::a]:24505","req_id":"8443b295-c66e-4fa4-bd43-8c410a7ac3af","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:13:55.770116039Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:43496"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:13:55.774113769Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"new_generation":1,"req_id":"34c19d32-b8ad-4fd7-97d0-7b9294da135a","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:13:55.77418343Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"new_generation":1,"req_id":"34c19d32-b8ad-4fd7-97d0-7b9294da135a","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:13:55.77419183Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"uri":"/config","method":"PUT","req_id":"34c19d32-b8ad-4fd7-97d0-7b9294da135a","remote_addr":"[fd00:1122:3344:101::a]:43496","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":3064,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:13:55.938532814Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:49070"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:13:55.938791689Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"uri":"/config","method":"GET","req_id":"bb612fa9-b29a-4dea-b385-306b474779f5","remote_addr":"[fd00:1122:3344:101::a]:49070","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":211,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 63450,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:13:56.556739749Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::a]:19742","req_id":"d3a28402-0371-4f8b-9b8a-4dba9b48557b","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:13:56.66957951Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:33909"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:13:56.670391226Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"uri":"/config","method":"GET","req_id":"f9896b5e-a874-43b3-a276-3d2f37524779","remote_addr":"[fd00:1122:3344:101::a]:33909","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":758,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 7401,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:14:04.205489181Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::b]:17273","req_id":"0464a4a5-6e0e-492e-a19d-d06b8e6c1f54","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52321,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:14:04.205639534Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::c]:15038","req_id":"656ebf18-e582-49ef-a30b-4dedcd698b0b","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:04.320445084Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:60074"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:04.320665658Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"uri":"/config","method":"GET","req_id":"259c3f72-fb52-4280-a134-33292d64999f","remote_addr":"[fd00:1122:3344:101::b]:60074","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":186,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:04.375613879Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:46486"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:04.375854704Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"uri":"/config","method":"GET","req_id":"9fb32b74-c045-47ba-a4f6-f13c7ee1bc02","remote_addr":"[fd00:1122:3344:101::c]:46486","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":191,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 37165,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:14:04.573672592Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::d]:55058","req_id":"f7bb8746-e34c-407b-b8a5-4a6c1f0de26c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 25010,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:14:14.57452229Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::d]:62382","req_id":"db2afb72-a8ac-4372-a139-1c9d2cf032a4","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 10438,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:14:16.4192284Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::1b]:49421","req_id":"eddb5dda-913e-4aea-a380-4b6e4bb02481","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 6910,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:14:16.42324692Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::1b]:57004","req_id":"c6445907-390a-4ade-89a9-6f93cf256c6c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46725,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:14:16.427349331Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::1b]:40298","req_id":"24665895-7822-4aab-ae88-85da066543be","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 36732,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:14:24.576249804Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::d]:56937","req_id":"3c6eeed2-c586-4c79-8537-b123339a5923","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43200,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:14:34.577776074Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::d]:58298","req_id":"1fd85b2a-49a3-46f6-a72b-6beff23cb3e6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 6155,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:14:44.579405626Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::d]:50566","req_id":"8c0a5f1c-59fb-4c41-98d2-bcb98b6d629f","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:51.871862483Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:63724"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:51.873047156Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"new_generation":1,"req_id":"23fa76c3-3f4e-49e0-b5f5-e83451fa798e","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:51.873090677Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"new_generation":1,"req_id":"23fa76c3-3f4e-49e0-b5f5-e83451fa798e","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:51.873099897Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"uri":"/config","method":"PUT","req_id":"23fa76c3-3f4e-49e0-b5f5-e83451fa798e","remote_addr":"[fd00:1122:3344:101::a]:63724","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":184,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:53.341890903Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:36118"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:53.343479205Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"new_generation":1,"req_id":"1ff4bed4-7fa4-4ccd-94db-964528a37151","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:53.343515665Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"new_generation":1,"req_id":"1ff4bed4-7fa4-4ccd-94db-964528a37151","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:53.343553676Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"uri":"/config","method":"PUT","req_id":"1ff4bed4-7fa4-4ccd-94db-964528a37151","remote_addr":"[fd00:1122:3344:101::c]:36118","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":284,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:53.453469379Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:43272"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:53.455283475Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"new_generation":1,"req_id":"2c581992-b3c2-4eeb-8ec4-27aea8082208","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:53.455338256Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"new_generation":1,"req_id":"2c581992-b3c2-4eeb-8ec4-27aea8082208","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:53.455348116Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"uri":"/config","method":"PUT","req_id":"2c581992-b3c2-4eeb-8ec4-27aea8082208","remote_addr":"[fd00:1122:3344:101::b]:43272","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":285,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54531,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:14:53.93295473Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::a]:27738","req_id":"211b8473-8be7-4e6b-8bb6-23b1c6eb30a5","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:54.031096948Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:65520"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:54.031327273Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"uri":"/config","method":"GET","req_id":"bfb1595e-df9d-4d25-a74d-905c152c8d48","remote_addr":"[fd00:1122:3344:101::a]:65520","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":194,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40753,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:14:54.038338282Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::c]:13306","req_id":"f2d99b42-f06f-42ee-a3d9-aae7273b2fea","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:54.137784137Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:52095"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:54.137999111Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"uri":"/config","method":"GET","req_id":"ff91c99d-bce4-4721-a049-9f1908975d1c","remote_addr":"[fd00:1122:3344:101::c]:52095","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":183,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 24791,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:14:54.230953887Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::b]:49078","req_id":"49095203-16b5-4ecd-8bc6-be680853e5b8","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:54.323440074Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:56904"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-04-10T03:14:54.323680778Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"uri":"/config","method":"GET","req_id":"8aa1baf4-27e6-4289-a053-ce616a0a19bc","remote_addr":"[fd00:1122:3344:101::b]:56904","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":204,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46171,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:14:54.580471288Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::d]:56018","req_id":"30b39fab-6efa-4e6d-83e9-c1f42bd8c17e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59379,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_943626f6-bfbc-4742-9c48-17f14e613b68.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_943626f6-bfbc-4742-9c48-17f14e613b68.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 57,\n 52,\n 51,\n 54,\n 50,\n 54,\n 102,\n 54,\n 45,\n 98,\n 102,\n 98,\n 99,\n 45,\n 52,\n 55,\n 52,\n 50,\n 45,\n 57,\n 99,\n 52,\n 56,\n 45,\n 49,\n 55,\n 102,\n 49,\n 52,\n 101,\n 54,\n 49,\n 51,\n 98,\n 54,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_943626f6-bfbc-4742-9c48-17f14e613b68.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:15:00.15648801Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::e]:47996","req_id":"c60f5ef7-fa54-45cd-8827-cb51723b91d1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59379,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_943626f6-bfbc-4742-9c48-17f14e613b68.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_943626f6-bfbc-4742-9c48-17f14e613b68.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 57,\n 52,\n 51,\n 54,\n 50,\n 54,\n 102,\n 54,\n 45,\n 98,\n 102,\n 98,\n 99,\n 45,\n 52,\n 55,\n 52,\n 50,\n 45,\n 57,\n 99,\n 52,\n 56,\n 45,\n 49,\n 55,\n 102,\n 49,\n 52,\n 101,\n 54,\n 49,\n 51,\n 98,\n 54,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_943626f6-bfbc-4742-9c48-17f14e613b68.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:15:00.15701485Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::e]:64502","req_id":"54885f63-4b68-4b2d-b14d-180194f158fd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4903,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_39112fc7-8025-4b5f-b87d-6c88d324f549.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_39112fc7-8025-4b5f-b87d-6c88d324f549.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 51,\n 57,\n 49,\n 49,\n 50,\n 102,\n 99,\n 55,\n 45,\n 56,\n 48,\n 50,\n 53,\n 45,\n 52,\n 98,\n 53,\n 102,\n 45,\n 98,\n 56,\n 55,\n 100,\n 45,\n 54,\n 99,\n 56,\n 56,\n 100,\n 51,\n 50,\n 52,\n 102,\n 53,\n 52,\n 57,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_39112fc7-8025-4b5f-b87d-6c88d324f549.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:15:00.800619061Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::1b]:51978","req_id":"ae22da73-d416-4f8e-822d-30ad8e5c5d1b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 39687,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_d3332475-1a75-4b57-9942-0d28202a50ca.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_d3332475-1a75-4b57-9942-0d28202a50ca.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 100,\n 51,\n 51,\n 51,\n 50,\n 52,\n 55,\n 53,\n 45,\n 49,\n 97,\n 55,\n 53,\n 45,\n 52,\n 98,\n 53,\n 55,\n 45,\n 57,\n 57,\n 52,\n 50,\n 45,\n 48,\n 100,\n 50,\n 56,\n 50,\n 48,\n 50,\n 97,\n 53,\n 48,\n 99,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_d3332475-1a75-4b57-9942-0d28202a50ca.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:15:00.930308616Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::3]:63097","req_id":"261f0f64-c7a8-4a10-bbc0-02281cb95cd5","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 39687,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_d3332475-1a75-4b57-9942-0d28202a50ca.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_d3332475-1a75-4b57-9942-0d28202a50ca.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 100,\n 51,\n 51,\n 51,\n 50,\n 52,\n 55,\n 53,\n 45,\n 49,\n 97,\n 55,\n 53,\n 45,\n 52,\n 98,\n 53,\n 55,\n 45,\n 57,\n 57,\n 52,\n 50,\n 45,\n 48,\n 100,\n 50,\n 56,\n 50,\n 48,\n 50,\n 97,\n 53,\n 48,\n 99,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_d3332475-1a75-4b57-9942-0d28202a50ca.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:15:00.930892348Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::3]:53052","req_id":"9512bd7f-b5e3-4bb3-a9f4-24ba259a45fd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20678,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_1d23e5c6-f4c0-4246-83aa-a514903668e9.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_1d23e5c6-f4c0-4246-83aa-a514903668e9.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 49,\n 100,\n 50,\n 51,\n 101,\n 53,\n 99,\n 54,\n 45,\n 102,\n 52,\n 99,\n 48,\n 45,\n 52,\n 50,\n 52,\n 54,\n 45,\n 56,\n 51,\n 97,\n 97,\n 45,\n 97,\n 53,\n 49,\n 52,\n 57,\n 48,\n 51,\n 54,\n 54,\n 56,\n 101,\n 57,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_1d23e5c6-f4c0-4246-83aa-a514903668e9.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:15:00.932088171Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::5]:48181","req_id":"434d02a8-4e8f-406f-bd58-ae171cb83b07","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20678,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_1d23e5c6-f4c0-4246-83aa-a514903668e9.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_1d23e5c6-f4c0-4246-83aa-a514903668e9.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 49,\n 100,\n 50,\n 51,\n 101,\n 53,\n 99,\n 54,\n 45,\n 102,\n 52,\n 99,\n 48,\n 45,\n 52,\n 50,\n 52,\n 54,\n 45,\n 56,\n 51,\n 97,\n 97,\n 45,\n 97,\n 53,\n 49,\n 52,\n 57,\n 48,\n 51,\n 54,\n 54,\n 56,\n 101,\n 57,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_1d23e5c6-f4c0-4246-83aa-a514903668e9.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:15:00.932713984Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::5]:59999","req_id":"46094b67-051f-46aa-aa78-a2b253dd06e0","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4357,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_050ac996-ad09-4979-9a3e-9d3e99d160e2.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_050ac996-ad09-4979-9a3e-9d3e99d160e2.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 48,\n 53,\n 48,\n 97,\n 99,\n 57,\n 57,\n 54,\n 45,\n 97,\n 100,\n 48,\n 57,\n 45,\n 52,\n 57,\n 55,\n 57,\n 45,\n 57,\n 97,\n 51,\n 101,\n 45,\n 57,\n 100,\n 51,\n 101,\n 57,\n 57,\n 100,\n 49,\n 54,\n 48,\n 101,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_050ac996-ad09-4979-9a3e-9d3e99d160e2.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:15:00.958846933Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::7]:49412","req_id":"f37c1bd9-36c1-470e-a4f9-0e1dc4d9c23b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4357,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_050ac996-ad09-4979-9a3e-9d3e99d160e2.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_050ac996-ad09-4979-9a3e-9d3e99d160e2.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 48,\n 53,\n 48,\n 97,\n 99,\n 57,\n 57,\n 54,\n 45,\n 97,\n 100,\n 48,\n 57,\n 45,\n 52,\n 57,\n 55,\n 57,\n 45,\n 57,\n 97,\n 51,\n 101,\n 45,\n 57,\n 100,\n 51,\n 101,\n 57,\n 57,\n 100,\n 49,\n 54,\n 48,\n 101,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_050ac996-ad09-4979-9a3e-9d3e99d160e2.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:15:00.959381113Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::7]:46236","req_id":"98e70b1d-d033-4401-8aa3-e172bc6bf311","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 47263,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_1d12c36e-bb96-4cf1-82ab-942e1c7f46ee.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_1d12c36e-bb96-4cf1-82ab-942e1c7f46ee.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 49,\n 100,\n 49,\n 50,\n 99,\n 51,\n 54,\n 101,\n 45,\n 98,\n 98,\n 57,\n 54,\n 45,\n 52,\n 99,\n 102,\n 49,\n 45,\n 56,\n 50,\n 97,\n 98,\n 45,\n 57,\n 52,\n 50,\n 101,\n 49,\n 99,\n 55,\n 102,\n 52,\n 54,\n 101,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_1d12c36e-bb96-4cf1-82ab-942e1c7f46ee.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:15:00.969762239Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::4]:50024","req_id":"ff8e97b3-8810-4012-bf22-b44ce65896e4","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 47263,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_1d12c36e-bb96-4cf1-82ab-942e1c7f46ee.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_1d12c36e-bb96-4cf1-82ab-942e1c7f46ee.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 49,\n 100,\n 49,\n 50,\n 99,\n 51,\n 54,\n 101,\n 45,\n 98,\n 98,\n 57,\n 54,\n 45,\n 52,\n 99,\n 102,\n 49,\n 45,\n 56,\n 50,\n 97,\n 98,\n 45,\n 57,\n 52,\n 50,\n 101,\n 49,\n 99,\n 55,\n 102,\n 52,\n 54,\n 101,\n 101,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_1d12c36e-bb96-4cf1-82ab-942e1c7f46ee.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:15:00.97027874Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::4]:47493","req_id":"1975def5-24e0-4f61-89bd-e6faeec178cc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54518,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_57eb77fc-97ef-4840-95c3-fe76535839dd.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_57eb77fc-97ef-4840-95c3-fe76535839dd.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 53,\n 55,\n 101,\n 98,\n 55,\n 55,\n 102,\n 99,\n 45,\n 57,\n 55,\n 101,\n 102,\n 45,\n 52,\n 56,\n 52,\n 48,\n 45,\n 57,\n 53,\n 99,\n 51,\n 45,\n 102,\n 101,\n 55,\n 54,\n 53,\n 51,\n 53,\n 56,\n 51,\n 57,\n 100,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_57eb77fc-97ef-4840-95c3-fe76535839dd.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:15:00.973662757Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::6]:54555","req_id":"6917513a-c0f7-40da-99a9-2d97a8ef88f3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54518,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_57eb77fc-97ef-4840-95c3-fe76535839dd.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_57eb77fc-97ef-4840-95c3-fe76535839dd.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 53,\n 55,\n 101,\n 98,\n 55,\n 55,\n 102,\n 99,\n 45,\n 57,\n 55,\n 101,\n 102,\n 45,\n 52,\n 56,\n 52,\n 48,\n 45,\n 57,\n 53,\n 99,\n 51,\n 45,\n 102,\n 101,\n 55,\n 54,\n 53,\n 51,\n 53,\n 56,\n 51,\n 57,\n 100,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_57eb77fc-97ef-4840-95c3-fe76535839dd.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:15:00.974189717Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::6]:64560","req_id":"9b5a36a1-804f-4fa3-b5be-8587f7f6708b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 39376,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:15:01.262043993Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::2]:64641","req_id":"ab7b9010-7d29-4cd8-bdb8-12ce04d41829","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 39376,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-10T03:15:01.262544833Z","hostname":"oxz_internal_dns_916b4819-7cc8-4b0c-ab85-e81e9faf89e7","pid":5576,"peer_addr":"[fd00:1122:3344:101::2]:56649","req_id":"f689fd3b-4e29-4833-ac4a-39d7000dda86","component":"dns"}