[ Mar 31 01:53:21 Enabled. ] [ Mar 31 01:53:21 Rereading configuration. ] [ Mar 31 01:53:21 Rereading configuration. ] [ Mar 31 01:53:22 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:3::1]:5353 --dns-address [fd00:1122:3344:3::1]:53 &"). ] [ Mar 31 01:53:22 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:53:22.840337698Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"dns_config":"Config { bind_address: [fd00:1122:3344:3::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:53:22.840518571Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:53:22.86232542Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:53:22.862385641Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:53:22.862442172Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"component":"dns","local_address":"[fd00:1122:3344:3::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:53:22.862735767Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:53:23.244625431Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:3::2]:34716"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:53:23.245462286Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"7ab5cbf8-ab04-45d2-8cdf-26b8287d7896","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:53:23.245492416Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:53:23.262700903Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:53:23.262727674Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"7ab5cbf8-ab04-45d2-8cdf-26b8287d7896","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:53:23.262734444Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"PUT","req_id":"7ab5cbf8-ab04-45d2-8cdf-26b8287d7896","remote_addr":"[fd00:1122:3344:3::2]:34716","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":17726,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14159,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:53:47.71576064Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::1b]:55692","req_id":"917ab498-345e-46ce-bea4-ebf690341f42","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54329,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:53:47.720326151Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::1b]:52573","req_id":"0b86c48d-d92c-4e14-8390-3c84f4e5e4c1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 19250,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:53:47.72420028Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::1b]:51910","req_id":"98ca0cd8-88c9-4a9e-be2b-3d06e694480e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43787,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:54:13.78712535Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::1b]:47502","req_id":"5c54782a-5f26-4058-9ff1-88868321c4a0","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34391,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:54:13.794250568Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::1b]:60620","req_id":"291cea2b-c147-45d2-b58e-ec6e65f9568a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 13735,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:54:13.794351279Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::1b]:45623","req_id":"3b99a401-dd1f-4061-b6b1-f12be0019812","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 26981,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:55:58.637596952Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:50581","req_id":"03b58c3f-abbb-4c07-93e9-4df3993e0015","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53423,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_9bdec556-0dc3-4de4-917b-6d5a06a3c9b2.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_9bdec556-0dc3-4de4-917b-6d5a06a3c9b2.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 57,\n 98,\n 100,\n 101,\n 99,\n 53,\n 53,\n 54,\n 45,\n 48,\n 100,\n 99,\n 51,\n 45,\n 52,\n 100,\n 101,\n 52,\n 45,\n 57,\n 49,\n 55,\n 98,\n 45,\n 54,\n 100,\n 53,\n 97,\n 48,\n 54,\n 97,\n 51,\n 99,\n 57,\n 98,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_9bdec556-0dc3-4de4-917b-6d5a06a3c9b2.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:56:02.707222237Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::e]:50461","req_id":"2b37a9f2-52e6-4660-b04b-eed95327c7e6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53423,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_9bdec556-0dc3-4de4-917b-6d5a06a3c9b2.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_9bdec556-0dc3-4de4-917b-6d5a06a3c9b2.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 57,\n 98,\n 100,\n 101,\n 99,\n 53,\n 53,\n 54,\n 45,\n 48,\n 100,\n 99,\n 51,\n 45,\n 52,\n 100,\n 101,\n 52,\n 45,\n 57,\n 49,\n 55,\n 98,\n 45,\n 54,\n 100,\n 53,\n 97,\n 48,\n 54,\n 97,\n 51,\n 99,\n 57,\n 98,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_9bdec556-0dc3-4de4-917b-6d5a06a3c9b2.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:56:02.70791936Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::e]:63283","req_id":"e05e29f1-0d7e-4cd2-8e24-8361d669406f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38289,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:56:12.907011962Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::b]:12112","req_id":"2a65ad9e-9945-4582-b3ca-c7696c2db692","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 39771,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:56:12.954204954Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::c]:3878","req_id":"607d05d0-eaa7-4847-8af4-c1b38e7749af","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 28313,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:56:13.040541855Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::a]:20489","req_id":"9e0f1a72-9769-4de0-be12-c894c5fe2577","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 6756,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:56:18.578175111Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:60038","req_id":"d8770fc4-e2f1-4f07-981c-701d1df01eb4","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 56504,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:56:28.579434973Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:52819","req_id":"69021d91-ffbf-45d0-b6ca-41e401016fe5","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 37280,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:56:38.580522053Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:57504","req_id":"0a65b5b9-c702-4b1f-bc83-499c9e6da7bc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46048,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:56:42.933843789Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::a]:5481","req_id":"d7754374-df26-4a85-b8b1-3294716a7ddc","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:56:43.178478534Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:63836"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:56:43.257607955Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"GET","req_id":"a10dcfb6-244b-4566-8cef-bf0d55f28c59","remote_addr":"[fd00:1122:3344:101::a]:63836","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":51544,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 25545,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:56:48.581792315Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:59517","req_id":"126d04ff-7c49-455f-a758-25f60614dc88","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:56:49.409424692Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:62375"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:56:49.413298541Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"24bed706-ff05-426b-82a2-701944e27a04","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:56:49.413324671Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"24bed706-ff05-426b-82a2-701944e27a04","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:56:49.413407893Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"PUT","req_id":"24bed706-ff05-426b-82a2-701944e27a04","remote_addr":"[fd00:1122:3344:101::c]:62375","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":237,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 65311,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:56:49.428212257Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::c]:49214","req_id":"d668889c-0246-4f30-9a8b-893c24b207f4","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:56:49.656233596Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:55167"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:56:49.656575342Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"GET","req_id":"d87b24b1-4fd8-4169-81a3-fc2ca5eb9085","remote_addr":"[fd00:1122:3344:101::c]:55167","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":293,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53487,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:56:50.369075715Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::c]:64522","req_id":"44d394bf-5863-484c-a347-f865096a362a","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:56:50.532212956Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:55258"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:56:50.532487671Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"GET","req_id":"8b11188d-fd23-4009-9a13-c4251d9cc8e4","remote_addr":"[fd00:1122:3344:101::c]:55258","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":214,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4780,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:56:53.154165819Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::a]:6241","req_id":"fdf9af39-0ce8-45c2-8813-272900f99ad5","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 22849,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:56:53.214051678Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::b]:9903","req_id":"0744aaf9-47cd-40c3-999f-24d2b8cd9bc7","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:56:53.28367572Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:51809"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:56:53.284099917Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"GET","req_id":"a9afdf7c-02c4-42d0-8ff9-de4a437a484a","remote_addr":"[fd00:1122:3344:101::a]:51809","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":360,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:56:53.378482732Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:45222"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:56:53.378766257Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"GET","req_id":"7bb3c819-4978-4879-9700-a52856db91e5","remote_addr":"[fd00:1122:3344:101::b]:45222","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":229,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 49668,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:56:58.58335607Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:61953","req_id":"93f2a6c0-7cd7-4924-a070-fbb6369c5551","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 24998,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:57:08.584896908Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:53661","req_id":"33b2666b-9f7b-49dc-92a8-7e6678772476","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:57:12.35678981Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:60250"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:57:12.360264802Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"ba247558-04c6-4fb3-a732-fd9dba28b5ef","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:57:12.360311793Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"ba247558-04c6-4fb3-a732-fd9dba28b5ef","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:57:12.360321943Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"PUT","req_id":"ba247558-04c6-4fb3-a732-fd9dba28b5ef","remote_addr":"[fd00:1122:3344:101::c]:60250","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":217,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:57:12.459986051Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:58216"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:57:12.464519912Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"50bae6df-a3b4-41ac-8a15-c26d44967691","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:57:12.464569533Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"50bae6df-a3b4-41ac-8a15-c26d44967691","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:57:12.464605513Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"PUT","req_id":"50bae6df-a3b4-41ac-8a15-c26d44967691","remote_addr":"[fd00:1122:3344:101::a]:58216","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":283,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:57:12.909124785Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:40681"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:57:12.914550832Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"35d8e570-3ca9-49c3-b794-e17bd81cf3b5","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:57:12.914608213Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"35d8e570-3ca9-49c3-b794-e17bd81cf3b5","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:57:12.914620073Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"PUT","req_id":"35d8e570-3ca9-49c3-b794-e17bd81cf3b5","remote_addr":"[fd00:1122:3344:101::b]:40681","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":181,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53318,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:57:13.540413309Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::a]:1254","req_id":"d0dd3bad-0d40-4100-802a-a598b50b4392","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 61277,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:57:13.540520991Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::b]:14087","req_id":"51965853-c7c8-47fb-b411-99b4ded47630","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20308,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:57:13.616348814Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::c]:39843","req_id":"d1829604-cc2e-4640-a703-dd207815cf9d","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:57:13.647578022Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:55120"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:57:13.647847426Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"GET","req_id":"12ac2ee5-1611-4b37-885e-6dc09591bf4e","remote_addr":"[fd00:1122:3344:101::a]:55120","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":201,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:57:13.688152465Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:44569"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:57:13.68841691Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"GET","req_id":"2a625aec-daed-4b11-b637-75b2e80d7454","remote_addr":"[fd00:1122:3344:101::b]:44569","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":229,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:57:13.721584542Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:48272"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:57:13.721821956Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"GET","req_id":"2e3f5253-9f7b-4658-a116-60c2484f9c5b","remote_addr":"[fd00:1122:3344:101::c]:48272","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":193,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 17851,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:57:18.587144598Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:49463","req_id":"d2463094-9071-409d-94d5-50380e8a7283","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 6403,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:57:28.588966191Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:54993","req_id":"fb2a73e2-6d48-46cf-9d5c-66c96c34ae33","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43190,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:57:38.590886174Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:59867","req_id":"913d601a-62de-4be8-83e4-1b3ea6d4d74f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50791,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:57:39.218145176Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::1b]:63269","req_id":"c23f2193-cc13-40c3-b15e-730465a641c6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21822,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:57:39.222720838Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::1b]:53803","req_id":"322df8b8-9ffc-46c5-ae95-dc5a78ff0847","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53305,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:57:39.226825271Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::1b]:65012","req_id":"7209e0c0-9c9b-4060-af93-2a8d89f17f98","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52800,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:57:48.592829989Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:59326","req_id":"e6f934ab-fd92-4532-aaab-0f12ac0ec350","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 25254,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:57:58.595343948Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:55181","req_id":"d7ac10a0-b324-45b6-9768-92be8eeb78b9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4848,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:58:08.597436516Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:49457","req_id":"90ecb18e-a9a1-4aa5-9a99-e81de485fa33","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:58:12.35712874Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:47425"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:58:12.357360154Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:35215"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:58:12.359668515Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"f556f0d2-ab69-4d1a-9f73-507b9b8097d2","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:58:12.359685185Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"f556f0d2-ab69-4d1a-9f73-507b9b8097d2","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:58:12.359729226Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"PUT","req_id":"f556f0d2-ab69-4d1a-9f73-507b9b8097d2","remote_addr":"[fd00:1122:3344:101::a]:47425","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":265,"response_code":204} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:58:12.360092133Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"eb182e6b-0689-43fa-8ff9-1cf964578998","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:58:12.360122153Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"eb182e6b-0689-43fa-8ff9-1cf964578998","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:58:12.360144743Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"PUT","req_id":"eb182e6b-0689-43fa-8ff9-1cf964578998","remote_addr":"[fd00:1122:3344:101::c]:35215","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":235,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:58:12.451692467Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:49346"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:58:12.454918825Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"3eeb967d-ed02-4561-aa32-3c3395799972","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:58:12.454957115Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"3eeb967d-ed02-4561-aa32-3c3395799972","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:58:12.455001716Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"PUT","req_id":"3eeb967d-ed02-4561-aa32-3c3395799972","remote_addr":"[fd00:1122:3344:101::b]:49346","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":252,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30046,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:58:13.457435072Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::c]:24353","req_id":"2d2d4d92-4bd3-474a-9ace-27136818f339","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3592,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:58:13.464828034Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::a]:52196","req_id":"184386fb-988b-43f4-99ea-ce536041d100","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 6267,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:58:13.524672162Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::b]:52272","req_id":"3538f52e-f484-4a55-ace1-0c62560c70e5","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:58:13.570690663Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:59984"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:58:13.570943958Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"GET","req_id":"8fd299c1-cab2-44f5-9b56-82a4158146ad","remote_addr":"[fd00:1122:3344:101::c]:59984","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":199,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:58:13.610466723Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:47148"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:58:13.610763768Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"GET","req_id":"88dcc068-5794-4ec4-be6b-4dc770bb014d","remote_addr":"[fd00:1122:3344:101::a]:47148","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":206,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:58:13.649858295Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:43953"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:58:13.65011309Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"GET","req_id":"32fd8d23-f173-40c7-a9cd-e99d0cfb73ed","remote_addr":"[fd00:1122:3344:101::b]:43953","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":217,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30595,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:58:18.59942012Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:49698","req_id":"a6fb5cd0-3099-45b2-8304-d787aa9acc40","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 51055,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:58:28.60115841Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:50363","req_id":"ff2542ae-c604-4b17-a677-4b5251ac5c10","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43360,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:58:38.603195473Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:52445","req_id":"07a0ff8d-9a92-45ec-bde0-577fd59d7f11","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59014,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:58:48.604877604Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:59924","req_id":"d5c23ec6-daf9-4a04-880f-279846878cf6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 45520,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:58:58.606589694Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:56562","req_id":"2b8dedf9-a0a6-426f-b39d-babc507b3c3d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 63573,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:59:08.608747062Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:55722","req_id":"9bc1422d-1cf5-48ad-bf42-c25b4c55b28c","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:59:12.357045733Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:48093"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:59:12.357280147Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:54990"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:59:12.359927104Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"bb3830f5-ee1e-45e1-8e9d-147eca80db46","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:59:12.359949585Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"bb3830f5-ee1e-45e1-8e9d-147eca80db46","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:59:12.359957675Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"PUT","req_id":"bb3830f5-ee1e-45e1-8e9d-147eca80db46","remote_addr":"[fd00:1122:3344:101::c]:48093","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":285,"response_code":204} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:59:12.36026632Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"0c8d419f-a980-4191-bce6-d1f7ea6ae146","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:59:12.360276381Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"0c8d419f-a980-4191-bce6-d1f7ea6ae146","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:59:12.360283661Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"PUT","req_id":"0c8d419f-a980-4191-bce6-d1f7ea6ae146","remote_addr":"[fd00:1122:3344:101::a]:54990","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":319,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:59:12.45157695Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:47950"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:59:12.455245285Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"ccc20873-2d4f-4b22-820e-24ca9799fa99","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:59:12.455285016Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"new_generation":1,"req_id":"ccc20873-2d4f-4b22-820e-24ca9799fa99","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:59:12.455297346Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"PUT","req_id":"ccc20873-2d4f-4b22-820e-24ca9799fa99","remote_addr":"[fd00:1122:3344:101::b]:47950","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":229,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 26618,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:59:13.463115149Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::a]:29839","req_id":"5d60e920-bfe1-475d-bcaa-16f8c55aa890","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 65031,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:59:13.472406485Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::b]:34050","req_id":"6145f051-4e63-4517-8999-dbb29fea4158","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 19814,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:59:13.52145974Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::c]:53296","req_id":"2b3de93f-9c30-4a27-8aaa-e2c285a6fb28","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:59:13.572402039Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:35723"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:59:13.572655983Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"GET","req_id":"5bb37b46-2f09-45ae-bdab-7350cea2b985","remote_addr":"[fd00:1122:3344:101::a]:35723","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":211,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:59:13.607723069Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:48581"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:59:13.607996854Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"GET","req_id":"d336e697-49fb-4a8a-aa23-5ea94eb75831","remote_addr":"[fd00:1122:3344:101::b]:48581","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":231,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:59:13.718788841Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:47372"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-31T01:59:13.719039055Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"uri":"/config","method":"GET","req_id":"5a0b7df0-a619-4884-8842-270ae350606b","remote_addr":"[fd00:1122:3344:101::c]:47372","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":199,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 15345,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:59:18.61083651Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:62438","req_id":"91758524-75db-459f-9a19-25188a20bb73","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34234,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:59:28.613217413Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:60980","req_id":"1962d6ef-2919-4c47-8257-f209a585feda","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 31422,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:59:38.615049085Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:50831","req_id":"9da6d153-7f45-4649-9e1f-56aacdfd1de6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16139,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:59:47.290958118Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::1b]:49314","req_id":"e2e35837-902c-4424-b279-80577496d430","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 26626,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:59:47.295048541Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::1b]:61671","req_id":"25fc3507-3527-466d-9fc9-8fd6ac13375e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 58243,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:59:47.299711034Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::1b]:65138","req_id":"0d5df5a7-d9fc-4045-b25b-87a5a8cffb12","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 350,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:59:48.616222285Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:62524","req_id":"a7c184ea-ec5f-446d-bb7a-cb380c11f437","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 48094,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-31T01:59:58.618061007Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::d]:63945","req_id":"2a91dd38-bf2c-44e2-8ce2-1f06492caffe","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62826,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_0a51eda0-0734-46ce-a0d6-2b72303777d2.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_0a51eda0-0734-46ce-a0d6-2b72303777d2.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 48,\n 97,\n 53,\n 49,\n 101,\n 100,\n 97,\n 48,\n 45,\n 48,\n 55,\n 51,\n 52,\n 45,\n 52,\n 54,\n 99,\n 101,\n 45,\n 97,\n 48,\n 100,\n 54,\n 45,\n 50,\n 98,\n 55,\n 50,\n 51,\n 48,\n 51,\n 55,\n 55,\n 55,\n 100,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_0a51eda0-0734-46ce-a0d6-2b72303777d2.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T02:00:00.047416941Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::6]:42242","req_id":"19d89ee8-6f77-4ea0-9aa1-23e60bfb4e99","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62826,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_0a51eda0-0734-46ce-a0d6-2b72303777d2.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_0a51eda0-0734-46ce-a0d6-2b72303777d2.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 48,\n 97,\n 53,\n 49,\n 101,\n 100,\n 97,\n 48,\n 45,\n 48,\n 55,\n 51,\n 52,\n 45,\n 52,\n 54,\n 99,\n 101,\n 45,\n 97,\n 48,\n 100,\n 54,\n 45,\n 50,\n 98,\n 55,\n 50,\n 51,\n 48,\n 51,\n 55,\n 55,\n 55,\n 100,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_0a51eda0-0734-46ce-a0d6-2b72303777d2.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T02:00:00.04790196Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::6]:35073","req_id":"915e988c-7618-4b8a-9ab1-0b7b739e8871","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 932,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_f985c274-e8e9-4c67-983e-52614781ed16.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_f985c274-e8e9-4c67-983e-52614781ed16.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 102,\n 57,\n 56,\n 53,\n 99,\n 50,\n 55,\n 52,\n 45,\n 101,\n 56,\n 101,\n 57,\n 45,\n 52,\n 99,\n 54,\n 55,\n 45,\n 57,\n 56,\n 51,\n 101,\n 45,\n 53,\n 50,\n 54,\n 49,\n 52,\n 55,\n 56,\n 49,\n 101,\n 100,\n 49,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_f985c274-e8e9-4c67-983e-52614781ed16.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T02:00:00.053293766Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::4]:38980","req_id":"82a3d219-977a-4967-a845-d31a05cda6cd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 932,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_f985c274-e8e9-4c67-983e-52614781ed16.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_f985c274-e8e9-4c67-983e-52614781ed16.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 102,\n 57,\n 56,\n 53,\n 99,\n 50,\n 55,\n 52,\n 45,\n 101,\n 56,\n 101,\n 57,\n 45,\n 52,\n 99,\n 54,\n 55,\n 45,\n 57,\n 56,\n 51,\n 101,\n 45,\n 53,\n 50,\n 54,\n 49,\n 52,\n 55,\n 56,\n 49,\n 101,\n 100,\n 49,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_f985c274-e8e9-4c67-983e-52614781ed16.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T02:00:00.053608372Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::4]:65430","req_id":"b351eef9-373d-42aa-9ed0-d82788edeb1e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 13631,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T02:00:00.323489287Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::2]:45806","req_id":"92d9aec0-7021-456b-8f4c-b58636046826","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 13631,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T02:00:00.323914364Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::2]:65509","req_id":"d3ffa203-3b81-4058-b229-c45946d12c94","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 7852,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_8bdce22a-3405-47bf-a1a8-a16662323e9a.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_8bdce22a-3405-47bf-a1a8-a16662323e9a.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 56,\n 98,\n 100,\n 99,\n 101,\n 50,\n 50,\n 97,\n 45,\n 51,\n 52,\n 48,\n 53,\n 45,\n 52,\n 55,\n 98,\n 102,\n 45,\n 97,\n 49,\n 97,\n 56,\n 45,\n 97,\n 49,\n 54,\n 54,\n 54,\n 50,\n 51,\n 50,\n 51,\n 101,\n 57,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_8bdce22a-3405-47bf-a1a8-a16662323e9a.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T02:00:00.383781803Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::1b]:45580","req_id":"b035d9d2-839c-4763-b67f-362414eedffb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21115,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_b77f9b5d-f5f1-4967-9206-7f6ef2571305.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_b77f9b5d-f5f1-4967-9206-7f6ef2571305.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 55,\n 55,\n 102,\n 57,\n 98,\n 53,\n 100,\n 45,\n 102,\n 53,\n 102,\n 49,\n 45,\n 52,\n 57,\n 54,\n 55,\n 45,\n 57,\n 50,\n 48,\n 54,\n 45,\n 55,\n 102,\n 54,\n 101,\n 102,\n 50,\n 53,\n 55,\n 49,\n 51,\n 48,\n 53,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_b77f9b5d-f5f1-4967-9206-7f6ef2571305.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T02:00:00.729272007Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::5]:35527","req_id":"043be7fb-1cfa-4753-9bfd-a98259c8fefd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21115,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_b77f9b5d-f5f1-4967-9206-7f6ef2571305.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_b77f9b5d-f5f1-4967-9206-7f6ef2571305.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 55,\n 55,\n 102,\n 57,\n 98,\n 53,\n 100,\n 45,\n 102,\n 53,\n 102,\n 49,\n 45,\n 52,\n 57,\n 54,\n 55,\n 45,\n 57,\n 50,\n 48,\n 54,\n 45,\n 55,\n 102,\n 54,\n 101,\n 102,\n 50,\n 53,\n 55,\n 49,\n 51,\n 48,\n 53,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_b77f9b5d-f5f1-4967-9206-7f6ef2571305.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T02:00:00.729853448Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::5]:48362","req_id":"76b49dbe-a95a-4ae6-b48b-a12c8143eb00","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34107,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_9bdec556-0dc3-4de4-917b-6d5a06a3c9b2.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_9bdec556-0dc3-4de4-917b-6d5a06a3c9b2.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 57,\n 98,\n 100,\n 101,\n 99,\n 53,\n 53,\n 54,\n 45,\n 48,\n 100,\n 99,\n 51,\n 45,\n 52,\n 100,\n 101,\n 52,\n 45,\n 57,\n 49,\n 55,\n 98,\n 45,\n 54,\n 100,\n 53,\n 97,\n 48,\n 54,\n 97,\n 51,\n 99,\n 57,\n 98,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_9bdec556-0dc3-4de4-917b-6d5a06a3c9b2.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T02:00:00.871328592Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::e]:58727","req_id":"fac08151-95ff-4691-8d7b-9df6f21020d7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34107,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_9bdec556-0dc3-4de4-917b-6d5a06a3c9b2.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_9bdec556-0dc3-4de4-917b-6d5a06a3c9b2.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 57,\n 98,\n 100,\n 101,\n 99,\n 53,\n 53,\n 54,\n 45,\n 48,\n 100,\n 99,\n 51,\n 45,\n 52,\n 100,\n 101,\n 52,\n 45,\n 57,\n 49,\n 55,\n 98,\n 45,\n 54,\n 100,\n 53,\n 97,\n 48,\n 54,\n 97,\n 51,\n 99,\n 57,\n 98,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_9bdec556-0dc3-4de4-917b-6d5a06a3c9b2.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T02:00:00.871829801Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::e]:52382","req_id":"4557ca9f-5b9d-4d71-8cc6-e1e39b5cd8ea","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 7628,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_43128b24-c24d-4545-a380-7108afaf8aa4.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_43128b24-c24d-4545-a380-7108afaf8aa4.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 52,\n 51,\n 49,\n 50,\n 56,\n 98,\n 50,\n 52,\n 45,\n 99,\n 50,\n 52,\n 100,\n 45,\n 52,\n 53,\n 52,\n 53,\n 45,\n 97,\n 51,\n 56,\n 48,\n 45,\n 55,\n 49,\n 48,\n 56,\n 97,\n 102,\n 97,\n 102,\n 56,\n 97,\n 97,\n 52,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_43128b24-c24d-4545-a380-7108afaf8aa4.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T02:00:00.949201001Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::3]:43296","req_id":"f6c938e5-b9dc-4efd-b29d-e4aa24f13717","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 7628,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_43128b24-c24d-4545-a380-7108afaf8aa4.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_43128b24-c24d-4545-a380-7108afaf8aa4.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 52,\n 51,\n 49,\n 50,\n 56,\n 98,\n 50,\n 52,\n 45,\n 99,\n 50,\n 52,\n 100,\n 45,\n 52,\n 53,\n 52,\n 53,\n 45,\n 97,\n 51,\n 56,\n 48,\n 45,\n 55,\n 49,\n 48,\n 56,\n 97,\n 102,\n 97,\n 102,\n 56,\n 97,\n 97,\n 52,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_43128b24-c24d-4545-a380-7108afaf8aa4.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T02:00:00.950130208Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::3]:56683","req_id":"4dbb5189-cfbf-4801-a968-86a7425126ed","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52699,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_3452532c-312e-48df-bdb2-a395d31bd418.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_3452532c-312e-48df-bdb2-a395d31bd418.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 52,\n 53,\n 50,\n 53,\n 51,\n 50,\n 99,\n 45,\n 51,\n 49,\n 50,\n 101,\n 45,\n 52,\n 56,\n 100,\n 102,\n 45,\n 98,\n 100,\n 98,\n 50,\n 45,\n 97,\n 51,\n 57,\n 53,\n 100,\n 51,\n 49,\n 98,\n 100,\n 52,\n 49,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_3452532c-312e-48df-bdb2-a395d31bd418.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T02:00:00.993033094Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::7]:32866","req_id":"6e6ff172-9557-4c97-bcf5-03f5bd0306e6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52699,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_3452532c-312e-48df-bdb2-a395d31bd418.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_3452532c-312e-48df-bdb2-a395d31bd418.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 52,\n 53,\n 50,\n 53,\n 51,\n 50,\n 99,\n 45,\n 51,\n 49,\n 50,\n 101,\n 45,\n 52,\n 56,\n 100,\n 102,\n 45,\n 98,\n 100,\n 98,\n 50,\n 45,\n 97,\n 51,\n 57,\n 53,\n 100,\n 51,\n 49,\n 98,\n 100,\n 52,\n 49,\n 56,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_3452532c-312e-48df-bdb2-a395d31bd418.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-31T02:00:00.993748027Z","hostname":"oxz_internal_dns_0d27c1e0-d5e7-42a6-9a50-50d26b407344","pid":5386,"peer_addr":"[fd00:1122:3344:101::7]:56928","req_id":"ecaf790c-4f0a-4eb4-8fbd-5577a3c2afcd","component":"dns"}