[ May 1 20:10:06 Enabled. ] [ May 1 20:10:06 Rereading configuration. ] [ May 1 20:10:07 Rereading configuration. ] [ May 1 20:10:10 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:2::1]:5353 --dns-address [fd00:1122:3344:2::1]:53 &"). ] [ May 1 20:10:10 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.003669516Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"dns_config":"Config { bind_address: [fd00:1122:3344:2::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [], compression: None }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.003801318Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.02186869Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.02188324Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.021944671Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"component":"dns","local_address":"[fd00:1122:3344:2::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.022152345Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.275915464Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:2::2]:44638"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.27677271Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"new_generation":1,"req_id":"2a1c9d7e-7721-4a04-bf7e-7af296c3e7b5","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.276792021Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.311055178Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.311086039Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"new_generation":1,"req_id":"2a1c9d7e-7721-4a04-bf7e-7af296c3e7b5","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:10:11.311109259Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"uri":"/config","method":"PUT","req_id":"2a1c9d7e-7721-4a04-bf7e-7af296c3e7b5","remote_addr":"[fd00:1122:3344:2::2]:44638","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":34798,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34251,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:10:24.320241129Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::1b]:38069","req_id":"6a8e74a3-f680-4c3c-ad03-1be525aacca3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50955,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:10:24.326047859Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::1b]:36410","req_id":"efc67aef-9b9c-479f-8042-0efb06f746b9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 24239,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:10:24.330147077Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::1b]:47986","req_id":"c90dc644-adcc-4616-922f-1457d8dc7aab","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 60531,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:12:28.049595939Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::1b]:61618","req_id":"5b424d30-18f6-4df0-a8ca-ed59556d1ef8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 51987,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:12:28.27194491Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::1b]:41151","req_id":"dcc4f12c-1492-4863-94b4-3661f62eff5b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 28316,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:12:28.287725989Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::1b]:63943","req_id":"af4a5dba-abac-4766-bce9-ca9f6a84ed0a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46123,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:30.746821528Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::d]:62907","req_id":"5672d005-8dc4-47a3-8e3c-1f6c8d8fd758","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 355,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_3d6bc9b3-122d-4a1b-bf2b-bedc362bf1dc.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_3d6bc9b3-122d-4a1b-bf2b-bedc362bf1dc.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 51,\n 100,\n 54,\n 98,\n 99,\n 57,\n 98,\n 51,\n 45,\n 49,\n 50,\n 50,\n 100,\n 45,\n 52,\n 97,\n 49,\n 98,\n 45,\n 98,\n 102,\n 50,\n 98,\n 45,\n 98,\n 101,\n 100,\n 99,\n 51,\n 54,\n 50,\n 98,\n 102,\n 49,\n 100,\n 99,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_3d6bc9b3-122d-4a1b-bf2b-bedc362bf1dc.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:34.540110484Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::e]:43334","req_id":"3a73770d-925b-41fb-9ab3-e337b67d398f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 355,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_3d6bc9b3-122d-4a1b-bf2b-bedc362bf1dc.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_3d6bc9b3-122d-4a1b-bf2b-bedc362bf1dc.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 51,\n 100,\n 54,\n 98,\n 99,\n 57,\n 98,\n 51,\n 45,\n 49,\n 50,\n 50,\n 100,\n 45,\n 52,\n 97,\n 49,\n 98,\n 45,\n 98,\n 102,\n 50,\n 98,\n 45,\n 98,\n 101,\n 100,\n 99,\n 51,\n 54,\n 50,\n 98,\n 102,\n 49,\n 100,\n 99,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_3d6bc9b3-122d-4a1b-bf2b-bedc362bf1dc.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:34.546798451Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::e]:59081","req_id":"157b2f34-2631-4018-9566-6075994970a2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53598,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:40.518531141Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::c]:61959","req_id":"264132fe-3136-4a70-9e2f-8266135ffd9e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14071,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:40.569004135Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::a]:27628","req_id":"1b3839c6-9663-4748-9ce5-74936d288145","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 22249,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:40.728880488Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::d]:61530","req_id":"3c42b70c-5458-4569-875f-dee23c44ce1e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62692,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:41.096783525Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::b]:49913","req_id":"6d74cd84-aba0-4593-9c98-9fd13390c37c","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:42.297910216Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:60544"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:42.352485668Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"new_generation":1,"req_id":"d01e52d1-a55a-44d6-976a-1de0a2278640","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:42.352590921Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"new_generation":1,"req_id":"d01e52d1-a55a-44d6-976a-1de0a2278640","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:42.352669782Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"uri":"/config","method":"PUT","req_id":"d01e52d1-a55a-44d6-976a-1de0a2278640","remote_addr":"[fd00:1122:3344:101::a]:60544","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":25180,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:42.504313589Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:56354"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:42.504393481Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:49127"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:42.504407891Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:41846"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:42.515269836Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"uri":"/config","method":"GET","req_id":"6a7bd903-d9c0-4cec-bdb6-b161fb19ffca","remote_addr":"[fd00:1122:3344:101::c]:49127","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":10765,"response_code":200} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:42.515349458Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"uri":"/config","method":"GET","req_id":"751c328e-f2b0-4639-8449-1024d82a2896","remote_addr":"[fd00:1122:3344:101::a]:56354","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":10895,"response_code":200} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:42.515402109Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"uri":"/config","method":"GET","req_id":"e5f89849-ee3b-4877-b031-2488d0affcf9","remote_addr":"[fd00:1122:3344:101::b]:41846","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":10727,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 63922,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:43.756389414Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::a]:1896","req_id":"9680c448-ee88-49f4-8524-6240b2cfeb85","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:43.88941532Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:63814"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:43.890031041Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"uri":"/config","method":"GET","req_id":"29686426-2ff7-4e8e-9a77-9d1250751ccd","remote_addr":"[fd00:1122:3344:101::a]:63814","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":223,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62703,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:44.535597428Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::a]:10283","req_id":"13698aaa-f668-4aff-98e7-661e3eab35b2","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:44.752217194Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:34029"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:44.7525148Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"uri":"/config","method":"GET","req_id":"94b01d0a-c80f-490a-b7f1-b2441a2700af","remote_addr":"[fd00:1122:3344:101::a]:34029","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":230,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 35485,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:45.218536081Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::a]:47788","req_id":"87bc041b-9f12-4038-95ab-d706a5746153","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:45.444053956Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:54805"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:45.4442807Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"uri":"/config","method":"GET","req_id":"d25ad326-40e6-4322-ad7c-2639cd6668cd","remote_addr":"[fd00:1122:3344:101::a]:54805","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":187,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 29925,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:50.73117218Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::d]:57649","req_id":"14a8ec5d-3ad6-498e-ba72-17a9840d2cab","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 275,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:51.295384009Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::c]:40749","req_id":"85ac980a-1f19-4425-869e-52c8a8d5568a","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:51.413287068Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:34648"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:51.413553993Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"uri":"/config","method":"GET","req_id":"e8586dfb-b0c2-4631-a09b-5c7c08833065","remote_addr":"[fd00:1122:3344:101::c]:34648","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":206,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 57170,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:13:51.875535108Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::b]:40112","req_id":"04efc539-2bad-4917-beee-21554acc1280","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:52.077489797Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:45841"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:13:52.077736412Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"uri":"/config","method":"GET","req_id":"582310d9-e197-4fb8-a3f2-f9ec679485e0","remote_addr":"[fd00:1122:3344:101::b]:45841","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":191,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4161,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:00.732548283Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::d]:56417","req_id":"f386e922-431a-4961-b9b3-85550b2cbbb6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4920,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:10.734474478Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::d]:54694","req_id":"01c28982-c37a-4b27-a157-7e64665d9d73","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43353,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:20.736319751Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::d]:52778","req_id":"1e2e3aad-700b-440a-96f6-6a3b1d11eefd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 32271,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:27.71504944Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::1b]:58357","req_id":"53230cb0-8082-4623-ba02-7ccba8c02066","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20704,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:27.719503374Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::1b]:51681","req_id":"2a762fcb-c191-4850-be5f-cf26c76b5398","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 51739,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:27.723913798Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::1b]:40804","req_id":"3402437e-de77-48c8-bd82-42639a8491ff","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59088,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:30.738330597Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::d]:61314","req_id":"2fc6f94d-b14b-4eb8-8e02-bd2158c7e03a","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:40.287333667Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:51051"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:40.288789165Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"new_generation":1,"req_id":"43b05886-a839-4f5f-a40e-46615e7f46a8","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:40.288829266Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"new_generation":1,"req_id":"43b05886-a839-4f5f-a40e-46615e7f46a8","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:40.288883576Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"uri":"/config","method":"PUT","req_id":"43b05886-a839-4f5f-a40e-46615e7f46a8","remote_addr":"[fd00:1122:3344:101::a]:51051","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":221,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:40.61644984Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:36804"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:40.617784535Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"new_generation":1,"req_id":"eef5df08-f3a5-4cc9-bcf3-f804209ec236","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:40.617849436Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"new_generation":1,"req_id":"eef5df08-f3a5-4cc9-bcf3-f804209ec236","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:40.617868317Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"uri":"/config","method":"PUT","req_id":"eef5df08-f3a5-4cc9-bcf3-f804209ec236","remote_addr":"[fd00:1122:3344:101::c]:36804","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":342,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41274,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:40.74021589Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::d]:50318","req_id":"e360f366-d012-4be2-8f2c-dc8683aba272","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:41.029339788Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:50011"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:41.03051824Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"new_generation":1,"req_id":"41330b8f-6fbf-46c8-8911-07ab9c531d27","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:41.03055708Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"new_generation":1,"req_id":"41330b8f-6fbf-46c8-8911-07ab9c531d27","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:41.030567971Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"uri":"/config","method":"PUT","req_id":"41330b8f-6fbf-46c8-8911-07ab9c531d27","remote_addr":"[fd00:1122:3344:101::b]:50011","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":270,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 36196,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:41.795872982Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::a]:15622","req_id":"7b7040eb-581e-4faa-9ed0-28fcf86d8cff","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3892,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:41.805466123Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::c]:33990","req_id":"9b75cbc6-f2a4-4d16-b594-55ab869e0841","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 32239,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:41.806010334Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::b]:13972","req_id":"fa48233c-0db5-4b35-9f42-81e449590aa5","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:41.921551158Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:37632"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:41.921786793Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"uri":"/config","method":"GET","req_id":"111cfa7c-cc8e-4e9b-bcf0-545a070ff3f4","remote_addr":"[fd00:1122:3344:101::c]:37632","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":198,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:41.990195806Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:63997"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:41.99041829Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"uri":"/config","method":"GET","req_id":"3ed44003-f090-48fb-b0e1-8574211940f5","remote_addr":"[fd00:1122:3344:101::a]:63997","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":189,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:42.029647982Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:35051"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-05-01T20:14:42.029887707Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"uri":"/config","method":"GET","req_id":"0ff465a1-829e-4665-adcf-18bb48316954","remote_addr":"[fd00:1122:3344:101::b]:35051","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":207,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23666,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:14:50.742303809Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::d]:63224","req_id":"a1e861a4-9145-4ce2-abd6-27842451c2e1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 50949,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_5b13b567-2d19-4d6f-b784-a4f69394fd82.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_5b13b567-2d19-4d6f-b784-a4f69394fd82.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 53,\n 98,\n 49,\n 51,\n 98,\n 53,\n 54,\n 55,\n 45,\n 50,\n 100,\n 49,\n 57,\n 45,\n 52,\n 100,\n 54,\n 102,\n 45,\n 98,\n 55,\n 56,\n 52,\n 45,\n 97,\n 52,\n 102,\n 54,\n 57,\n 51,\n 57,\n 52,\n 102,\n 100,\n 56,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_5b13b567-2d19-4d6f-b784-a4f69394fd82.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:15:00.123506066Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::1b]:44574","req_id":"3c52186b-855f-4d94-9731-c3a82ae9bb27","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14372,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:15:00.282104585Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::2]:51668","req_id":"fb591e3e-ab17-4496-9e2f-696cabc2eb04","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14372,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-05-01T20:15:00.282711747Z","hostname":"oxz_internal_dns_24ed7992-b47f-4d02-ba23-dc51b9cd8bb0","pid":5553,"peer_addr":"[fd00:1122:3344:101::2]:46279","req_id":"427edef5-1137-4ccc-a081-1a6d4bf69285","component":"dns"}