[ Mar 24 17:53:42 Enabled. ] [ Mar 24 17:53:42 Rereading configuration. ] [ Mar 24 17:53:43 Rereading configuration. ] [ Mar 24 17:53:44 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:1::1]:5353 --dns-address [fd00:1122:3344:1::1]:53 &"). ] [ Mar 24 17:53:44 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:1::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:44.335774325Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"dns_config":"Config { bind_address: [fd00:1122:3344:1::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:1::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:44.335949639Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:44.354263317Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:44.354281927Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:44.354324908Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"component":"dns","local_address":"[fd00:1122:3344:1::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:44.354621316Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:45.955715003Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:1::2]:51641"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:45.956537204Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"new_generation":1,"req_id":"fa709fa5-2546-4ff2-a998-147f078625f3","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:45.956550194Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:45.975504828Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:45.975527699Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"new_generation":1,"req_id":"fa709fa5-2546-4ff2-a998-147f078625f3","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:45.975545939Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"uri":"/config","method":"PUT","req_id":"fa709fa5-2546-4ff2-a998-147f078625f3","remote_addr":"[fd00:1122:3344:1::2]:51641","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":19392,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59310,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:53:57.846605542Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:57810","req_id":"8a5e24bb-21fe-4c87-b9c1-360f7483bc85","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16257,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:53:57.851892507Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:65074","req_id":"1317266a-41ad-497b-bf86-fc3c1781e962","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40028,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:53:57.856838523Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:57240","req_id":"f98645b2-68a8-4ca7-9cac-84b30c1a96a8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 33516,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:55:58.680141076Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:42665","req_id":"b8042e73-a7b0-4a92-b7fc-136ff398e865","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 29842,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:55:58.684479722Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:58288","req_id":"7093ee19-7612-4151-96c3-7f2c0f6990a2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 11603,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:55:58.689135403Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:64518","req_id":"2e418c84-17f6-453c-b765-4b261b0c7506","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34793,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:56:32.449374049Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:34648","req_id":"44e7064f-42b0-4ff6-a0ae-de8a204cd4ed","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3211,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:56:32.453278515Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:64361","req_id":"de5c1788-0a8b-4e1a-bd01-c7a4008aa3b0","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23891,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:56:32.457272407Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:38330","req_id":"abb5ecb2-8ad3-40f6-bf38-c87bd1c449ef","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53464,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:42.04960253Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:55700","req_id":"36747da3-0aa6-42cf-8c52-1b693f8369a1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18809,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 51,\n 48,\n 55,\n 102,\n 53,\n 56,\n 51,\n 52,\n 45,\n 51,\n 50,\n 51,\n 54,\n 45,\n 52,\n 48,\n 101,\n 48,\n 45,\n 56,\n 51,\n 51,\n 52,\n 45,\n 100,\n 55,\n 56,\n 57,\n 97,\n 52,\n 48,\n 48,\n 52,\n 101,\n 99,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:47.984156766Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::e]:51048","req_id":"60525769-a52a-444a-8a08-e234935b9c3b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18809,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 51,\n 48,\n 55,\n 102,\n 53,\n 56,\n 51,\n 52,\n 45,\n 51,\n 50,\n 51,\n 54,\n 45,\n 52,\n 48,\n 101,\n 48,\n 45,\n 56,\n 51,\n 51,\n 52,\n 45,\n 100,\n 55,\n 56,\n 57,\n 97,\n 52,\n 48,\n 48,\n 52,\n 101,\n 99,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:48.00936563Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::e]:49466","req_id":"ea877844-3efc-48f2-ab0b-00087ea58e73","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1040,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:52.022422286Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:64916","req_id":"a7c8ae16-1533-4a4b-99d4-fa28eaa07b48","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 19875,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:57.70006697Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::a]:26152","req_id":"24b7b8fa-2848-481b-8c6c-5a2dc9c9e293","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 637,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:57.700227754Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::c]:34887","req_id":"e7a6438b-9a36-4d4c-87bf-46f69d257ccd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4046,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:57.969523253Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::b]:2036","req_id":"b9e11d74-b78f-4bda-9e0f-0e8b6392bcc4","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54156,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:02.024288534Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:51872","req_id":"e85f6bed-da50-423a-8f01-b517855bf0b3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38249,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:12.025840056Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:56800","req_id":"13ed0abc-6a3d-4dcb-9c99-4f7154900f33","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 7385,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:19.969912412Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::a]:15839","req_id":"4a577bd5-ce9e-4cab-bb19-4f67e35f6eae","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:20.339157044Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:54384"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:20.387984431Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"uri":"/config","method":"GET","req_id":"26ba1a73-bedb-4f27-aeaf-5e9896976e5e","remote_addr":"[fd00:1122:3344:101::a]:54384","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":23412,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41350,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:22.027032378Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:65361","req_id":"16644036-74df-42cc-9e7b-78052cdac22d","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:24.633515295Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:39988"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:24.634370137Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"new_generation":1,"req_id":"a559d5be-35e7-4d5b-96a8-fcbb5027b33a","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:24.634391958Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"new_generation":1,"req_id":"a559d5be-35e7-4d5b-96a8-fcbb5027b33a","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:24.634450719Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"uri":"/config","method":"PUT","req_id":"a559d5be-35e7-4d5b-96a8-fcbb5027b33a","remote_addr":"[fd00:1122:3344:101::b]:39988","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":780,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34752,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:24.767882917Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::b]:52981","req_id":"993f11e6-6e86-422e-9952-85eb162f4ad6","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:24.916092723Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:36213"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:24.916413561Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"uri":"/config","method":"GET","req_id":"00f8480d-ca42-4ef6-a451-4fb3898aee09","remote_addr":"[fd00:1122:3344:101::b]:36213","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":264,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 58947,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:25.789659216Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::b]:20687","req_id":"e00a327e-6397-4b2f-9297-250268106650","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:25.949090339Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:52830"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:25.949497039Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"uri":"/config","method":"GET","req_id":"8b221de8-3d16-43bf-919f-ca2edf2c7637","remote_addr":"[fd00:1122:3344:101::b]:52830","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":261,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30285,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:28.384983489Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::a]:46980","req_id":"5280d620-4fee-4261-b5f2-9f81665a452e","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:28.502962843Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:51655"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:28.503231269Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"uri":"/config","method":"GET","req_id":"59713f10-2781-4d5d-9e56-7500928a370a","remote_addr":"[fd00:1122:3344:101::a]:51655","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":212,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 29721,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:28.8500947Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::c]:26833","req_id":"f5ba6a57-111e-42f5-8303-d377cd0d2aa0","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:28.98830761Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:43809"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:28.988615498Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"uri":"/config","method":"GET","req_id":"ce002d93-6b5a-4b24-9a13-25204f075944","remote_addr":"[fd00:1122:3344:101::c]:43809","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":253,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 63510,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:32.028540437Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:57873","req_id":"dbe42a6f-2a84-4df4-b969-8385b5432fdb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1296,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:42.029607876Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:61367","req_id":"ffb790af-1482-40b8-a7e1-eed38b2a1025","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 9785,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:50.792034376Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:58033","req_id":"9a6a49ea-cfce-4963-a573-8b947bbe32a3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64867,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:50.80468891Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:36344","req_id":"eeaeae13-ee41-4d21-a17e-a1a9a4de8c8a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53351,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:50.809676137Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:45494","req_id":"4e298a90-43de-4911-85de-2acca46cae1f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 13167,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:52.03085361Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:57276","req_id":"f6c655c2-64ff-42eb-991c-7068030cd147","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.544948997Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:39169"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.545288676Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"new_generation":1,"req_id":"a9a4dbeb-d064-4adf-9b93-a9377f5900a6","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.545338437Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"new_generation":1,"req_id":"a9a4dbeb-d064-4adf-9b93-a9377f5900a6","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.545350417Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"uri":"/config","method":"PUT","req_id":"a9a4dbeb-d064-4adf-9b93-a9377f5900a6","remote_addr":"[fd00:1122:3344:101::a]:39169","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":284,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.673750687Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:50479"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.674083236Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"new_generation":1,"req_id":"f59915a2-c307-4980-b7f4-2002765ae9c6","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.674095756Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"new_generation":1,"req_id":"f59915a2-c307-4980-b7f4-2002765ae9c6","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.674109706Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"uri":"/config","method":"PUT","req_id":"f59915a2-c307-4980-b7f4-2002765ae9c6","remote_addr":"[fd00:1122:3344:101::c]:50479","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":213,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.731394589Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:51723"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.731868122Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"new_generation":1,"req_id":"3c695a51-40b8-45b7-8050-f91927061a1a","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.731921453Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"new_generation":1,"req_id":"3c695a51-40b8-45b7-8050-f91927061a1a","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.731931013Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"uri":"/config","method":"PUT","req_id":"3c695a51-40b8-45b7-8050-f91927061a1a","remote_addr":"[fd00:1122:3344:101::b]:51723","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":273,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 17410,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:58.690430066Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::c]:35100","req_id":"4afd4be7-31af-43cc-b210-a109cce20173","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21156,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:58.696005799Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::b]:13117","req_id":"3d1693ef-4c05-4554-b3d1-7855c511b53a","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:58.798675861Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:52294"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:58.798904057Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"uri":"/config","method":"GET","req_id":"dcf8febb-ea7f-449e-91cf-5eaf9d1ed246","remote_addr":"[fd00:1122:3344:101::c]:52294","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":189,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:58.829698174Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:38962"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:58.82995514Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"uri":"/config","method":"GET","req_id":"0b80d9f5-8c82-4cdd-be81-7454f7964065","remote_addr":"[fd00:1122:3344:101::b]:38962","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":216,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16865,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.050074766Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::a]:53264","req_id":"d3d95ff4-0a54-4b14-b3c6-47ee2074aafc","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T18:00:00.145140334Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:45204"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T18:00:00.145411281Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"uri":"/config","method":"GET","req_id":"eab7815e-2d04-4514-89c3-d7150949f324","remote_addr":"[fd00:1122:3344:101::a]:45204","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":226,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 17964,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_382def60-3e0e-43da-b798-0e943c074eea.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_382def60-3e0e-43da-b798-0e943c074eea.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 51,\n 56,\n 50,\n 100,\n 101,\n 102,\n 54,\n 48,\n 45,\n 51,\n 101,\n 48,\n 101,\n 45,\n 52,\n 51,\n 100,\n 97,\n 45,\n 98,\n 55,\n 57,\n 56,\n 45,\n 48,\n 101,\n 57,\n 52,\n 51,\n 99,\n 48,\n 55,\n 52,\n 101,\n 101,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_382def60-3e0e-43da-b798-0e943c074eea.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.382377804Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::1b]:36108","req_id":"b7fb4a03-13c8-44a3-91b3-ef7b6c44b4e6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 12386,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_a1cd5147-89ba-44eb-9348-15341068db10.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_a1cd5147-89ba-44eb-9348-15341068db10.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 97,\n 49,\n 99,\n 100,\n 53,\n 49,\n 52,\n 55,\n 45,\n 56,\n 57,\n 98,\n 97,\n 45,\n 52,\n 52,\n 101,\n 98,\n 45,\n 57,\n 51,\n 52,\n 56,\n 45,\n 49,\n 53,\n 51,\n 52,\n 49,\n 48,\n 54,\n 56,\n 100,\n 98,\n 49,\n 48,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_a1cd5147-89ba-44eb-9348-15341068db10.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.601737847Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::6]:58174","req_id":"34203f46-366b-4d4f-9fc6-42de5f0c6490","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 12386,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_a1cd5147-89ba-44eb-9348-15341068db10.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_a1cd5147-89ba-44eb-9348-15341068db10.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 97,\n 49,\n 99,\n 100,\n 53,\n 49,\n 52,\n 55,\n 45,\n 56,\n 57,\n 98,\n 97,\n 45,\n 52,\n 52,\n 101,\n 98,\n 45,\n 57,\n 51,\n 52,\n 56,\n 45,\n 49,\n 53,\n 51,\n 52,\n 49,\n 48,\n 54,\n 56,\n 100,\n 98,\n 49,\n 48,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_a1cd5147-89ba-44eb-9348-15341068db10.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.602352813Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::6]:41832","req_id":"345dae05-db33-4841-9d5d-ae02e8259bf7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 24817,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 51,\n 48,\n 55,\n 102,\n 53,\n 56,\n 51,\n 52,\n 45,\n 51,\n 50,\n 51,\n 54,\n 45,\n 52,\n 48,\n 101,\n 48,\n 45,\n 56,\n 51,\n 51,\n 52,\n 45,\n 100,\n 55,\n 56,\n 57,\n 97,\n 52,\n 48,\n 48,\n 52,\n 101,\n 99,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.607192167Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::e]:60842","req_id":"91f1464d-1e0b-4a48-ab09-0e5daf298c15","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 24817,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 51,\n 48,\n 55,\n 102,\n 53,\n 56,\n 51,\n 52,\n 45,\n 51,\n 50,\n 51,\n 54,\n 45,\n 52,\n 48,\n 101,\n 48,\n 45,\n 56,\n 51,\n 51,\n 52,\n 45,\n 100,\n 55,\n 56,\n 57,\n 97,\n 52,\n 48,\n 48,\n 52,\n 101,\n 99,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.60772764Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::e]:46748","req_id":"0d7755ef-d798-4887-bf42-0de148d15f0e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 13560,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_39c38aea-dee6-4629-8092-5ab4ce37b2cb.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_39c38aea-dee6-4629-8092-5ab4ce37b2cb.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 57,\n 99,\n 51,\n 56,\n 97,\n 101,\n 97,\n 45,\n 100,\n 101,\n 101,\n 54,\n 45,\n 52,\n 54,\n 50,\n 57,\n 45,\n 56,\n 48,\n 57,\n 50,\n 45,\n 53,\n 97,\n 98,\n 52,\n 99,\n 101,\n 51,\n 55,\n 98,\n 50,\n 99,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_39c38aea-dee6-4629-8092-5ab4ce37b2cb.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.657947663Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::4]:65408","req_id":"15347866-756c-4493-8c13-3f56714ab91c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 13560,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_39c38aea-dee6-4629-8092-5ab4ce37b2cb.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_39c38aea-dee6-4629-8092-5ab4ce37b2cb.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 57,\n 99,\n 51,\n 56,\n 97,\n 101,\n 97,\n 45,\n 100,\n 101,\n 101,\n 54,\n 45,\n 52,\n 54,\n 50,\n 57,\n 45,\n 56,\n 48,\n 57,\n 50,\n 45,\n 53,\n 97,\n 98,\n 52,\n 99,\n 101,\n 51,\n 55,\n 98,\n 50,\n 99,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_39c38aea-dee6-4629-8092-5ab4ce37b2cb.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.658742494Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::4]:43114","req_id":"3efd306f-33a3-4b55-987e-6ea05062a51f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43436,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_39e05c87-abdd-4311-accb-06499195f5f5.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_39e05c87-abdd-4311-accb-06499195f5f5.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 57,\n 101,\n 48,\n 53,\n 99,\n 56,\n 55,\n 45,\n 97,\n 98,\n 100,\n 100,\n 45,\n 52,\n 51,\n 49,\n 49,\n 45,\n 97,\n 99,\n 99,\n 98,\n 45,\n 48,\n 54,\n 52,\n 57,\n 57,\n 49,\n 57,\n 53,\n 102,\n 53,\n 102,\n 53,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_39e05c87-abdd-4311-accb-06499195f5f5.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.65976909Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::7]:60284","req_id":"f5ac6cb5-9143-43bd-9e01-83934b841650","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43436,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_39e05c87-abdd-4311-accb-06499195f5f5.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_39e05c87-abdd-4311-accb-06499195f5f5.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 57,\n 101,\n 48,\n 53,\n 99,\n 56,\n 55,\n 45,\n 97,\n 98,\n 100,\n 100,\n 45,\n 52,\n 51,\n 49,\n 49,\n 45,\n 97,\n 99,\n 99,\n 98,\n 45,\n 48,\n 54,\n 52,\n 57,\n 57,\n 49,\n 57,\n 53,\n 102,\n 53,\n 102,\n 53,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_39e05c87-abdd-4311-accb-06499195f5f5.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.660329374Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::7]:63237","req_id":"436b0309-2efb-46c2-a405-e6487bff7cad","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1949,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_9e155ee0-5cdf-431a-b548-2719bc3491af.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_9e155ee0-5cdf-431a-b548-2719bc3491af.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 57,\n 101,\n 49,\n 53,\n 53,\n 101,\n 101,\n 48,\n 45,\n 53,\n 99,\n 100,\n 102,\n 45,\n 52,\n 51,\n 49,\n 97,\n 45,\n 98,\n 53,\n 52,\n 56,\n 45,\n 50,\n 55,\n 49,\n 57,\n 98,\n 99,\n 51,\n 52,\n 57,\n 49,\n 97,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_9e155ee0-5cdf-431a-b548-2719bc3491af.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.671936471Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::5]:60563","req_id":"356c02cd-36bc-4a08-94d1-fd616ada2206","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1949,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_9e155ee0-5cdf-431a-b548-2719bc3491af.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_9e155ee0-5cdf-431a-b548-2719bc3491af.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 57,\n 101,\n 49,\n 53,\n 53,\n 101,\n 101,\n 48,\n 45,\n 53,\n 99,\n 100,\n 102,\n 45,\n 52,\n 51,\n 49,\n 97,\n 45,\n 98,\n 53,\n 52,\n 56,\n 45,\n 50,\n 55,\n 49,\n 57,\n 98,\n 99,\n 51,\n 52,\n 57,\n 49,\n 97,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_9e155ee0-5cdf-431a-b548-2719bc3491af.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.672471424Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::5]:52723","req_id":"0974228f-2948-44ac-b80a-6061ef60bd15","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23681,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.708020862Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::2]:36337","req_id":"6c6012bf-353b-4ac0-b046-e8128fcd6f85","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23681,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.708667009Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::2]:37633","req_id":"3e8ea93e-b2b7-4147-ae17-1f38e5d2cec2","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41484,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_70133477-80a7-453e-9b7f-ec784ac8ce06.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_70133477-80a7-453e-9b7f-ec784ac8ce06.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 55,\n 48,\n 49,\n 51,\n 51,\n 52,\n 55,\n 55,\n 45,\n 56,\n 48,\n 97,\n 55,\n 45,\n 52,\n 53,\n 51,\n 101,\n 45,\n 57,\n 98,\n 55,\n 102,\n 45,\n 101,\n 99,\n 55,\n 56,\n 52,\n 97,\n 99,\n 56,\n 99,\n 101,\n 48,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_70133477-80a7-453e-9b7f-ec784ac8ce06.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.737948447Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::3]:36028","req_id":"3bf689b1-de82-4db0-9309-38a356cbc4ec","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41484,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_70133477-80a7-453e-9b7f-ec784ac8ce06.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_70133477-80a7-453e-9b7f-ec784ac8ce06.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 55,\n 48,\n 49,\n 51,\n 51,\n 52,\n 55,\n 55,\n 45,\n 56,\n 48,\n 97,\n 55,\n 45,\n 52,\n 53,\n 51,\n 101,\n 45,\n 57,\n 98,\n 55,\n 102,\n 45,\n 101,\n 99,\n 55,\n 56,\n 52,\n 97,\n 99,\n 56,\n 99,\n 101,\n 48,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_70133477-80a7-453e-9b7f-ec784ac8ce06.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.738688796Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::3]:61884","req_id":"f69e51a8-3077-45a0-b11d-a59064261cb5","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59140,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:02.031347995Z","hostname":"oxz_internal_dns_b89906d2-0c7e-4a7f-a5f8-90deda2c09d2","pid":5492,"peer_addr":"[fd00:1122:3344:101::d]:62629","req_id":"d82cb01d-1c55-4122-996d-89601e8d2e6d","component":"dns"}