[ Mar 24 17:53:42 Enabled. ] [ Mar 24 17:53:42 Rereading configuration. ] [ Mar 24 17:53:42 Rereading configuration. ] [ Mar 24 17:53:44 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:2::1]:5353 --dns-address [fd00:1122:3344:2::1]:53 &"). ] [ Mar 24 17:53:44 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:44.258540652Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"dns_config":"Config { bind_address: [fd00:1122:3344:2::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:2::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:44.258684436Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:44.277155917Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:44.277165677Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:44.277219669Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"component":"dns","local_address":"[fd00:1122:3344:2::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:44.277512937Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:45.93640943Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:2::2]:55400"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:45.937317283Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"new_generation":1,"req_id":"a0b45960-debb-4d75-ae9c-953cc40d7bba","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:45.937330363Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:45.954383069Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:45.95440755Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"new_generation":1,"req_id":"a0b45960-debb-4d75-ae9c-953cc40d7bba","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:45.95442469Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"uri":"/config","method":"PUT","req_id":"a0b45960-debb-4d75-ae9c-953cc40d7bba","remote_addr":"[fd00:1122:3344:2::2]:55400","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":17522,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59310,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:53:57.847619798Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::1b]:35639","req_id":"e7663e12-2957-4eea-92d4-ad9cfee03bc7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16257,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:53:57.852067252Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::1b]:52898","req_id":"e66ee4dc-221f-4769-84c0-5c99c5c4fec8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40028,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:53:57.857004468Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::1b]:58411","req_id":"395413a7-b1cb-4115-ad93-ffbf60d83800","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 33516,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:55:58.680383988Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::1b]:59288","req_id":"a3a65cd5-824c-47a5-8f2d-9360228b48d9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 29842,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:55:58.684749166Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::1b]:46764","req_id":"ad3c8239-7568-4004-916b-660f570ff544","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 11603,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:55:58.689469396Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::1b]:54583","req_id":"b33fbf01-c0c9-4b7a-9ba9-8268f2e70416","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34793,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:56:32.44955966Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::1b]:37094","req_id":"bfbc0066-7c6c-4e7a-a907-26b2c956e0e8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3211,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:56:32.453415215Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::1b]:49959","req_id":"83102759-b920-470e-ac5a-0797a258a836","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23891,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:56:32.457444264Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::1b]:43359","req_id":"81378c99-130f-41af-b243-313a131d13af","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 25074,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:42.063392743Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::d]:56538","req_id":"96e7f4ac-8381-4ed4-865b-bbc1cd298b6c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18809,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 51,\n 48,\n 55,\n 102,\n 53,\n 56,\n 51,\n 52,\n 45,\n 51,\n 50,\n 51,\n 54,\n 45,\n 52,\n 48,\n 101,\n 48,\n 45,\n 56,\n 51,\n 51,\n 52,\n 45,\n 100,\n 55,\n 56,\n 57,\n 97,\n 52,\n 48,\n 48,\n 52,\n 101,\n 99,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:47.991897164Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::e]:61732","req_id":"151f0d90-fcc5-47ee-b3aa-59c9875c93cb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18809,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 51,\n 48,\n 55,\n 102,\n 53,\n 56,\n 51,\n 52,\n 45,\n 51,\n 50,\n 51,\n 54,\n 45,\n 52,\n 48,\n 101,\n 48,\n 45,\n 56,\n 51,\n 51,\n 52,\n 45,\n 100,\n 55,\n 56,\n 57,\n 97,\n 52,\n 48,\n 48,\n 52,\n 101,\n 99,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:48.010100829Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::e]:64052","req_id":"ad0f78e7-effb-4911-903c-e26460848662","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 62749,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:52.022519898Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::d]:49269","req_id":"1be49956-3904-4e3a-afdd-51ce65b37473","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 47518,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:57.700281666Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::a]:19829","req_id":"a0d9ce37-4739-479a-a95c-89c71102a480","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 4655,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:57.700493901Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::c]:52010","req_id":"93e93132-5039-4b8c-a96e-4a5c073f4456","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18277,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:57.969617116Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::b]:65116","req_id":"f3d0127c-efa1-435f-ae23-7e31349ef8cc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59270,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:02.024272174Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::d]:54167","req_id":"bac0a3c9-e57a-4945-99fb-3ea865ed04c8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 37565,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:12.025875686Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::d]:60729","req_id":"8d5ed398-cc3f-42b7-ac91-5c862bb590ac","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 26588,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:19.969864071Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::a]:62644","req_id":"38366574-c6ea-4599-abb9-ebef95eed1b1","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:20.278091104Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:58049"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:20.329530608Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"uri":"/config","method":"GET","req_id":"1f0abdb0-e4f0-4fef-9c63-b2a6a0e9439b","remote_addr":"[fd00:1122:3344:101::a]:58049","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":25965,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 19434,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:22.027071579Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::d]:64916","req_id":"6bf06cc6-07aa-4738-9680-7133230ff15b","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:24.632593621Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:42792"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:24.633794022Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"new_generation":1,"req_id":"d6426025-ac9d-44e7-9b8f-4caee0790526","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:24.633808363Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"new_generation":1,"req_id":"d6426025-ac9d-44e7-9b8f-4caee0790526","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:24.633858844Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"uri":"/config","method":"PUT","req_id":"d6426025-ac9d-44e7-9b8f-4caee0790526","remote_addr":"[fd00:1122:3344:101::b]:42792","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":255,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59440,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:24.767704383Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::b]:21935","req_id":"c5a63c61-8f15-4cfb-910c-415f4158991d","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:24.915436056Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:52189"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:24.915678523Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"uri":"/config","method":"GET","req_id":"93026622-54c8-4f0e-a413-807cf1f338ca","remote_addr":"[fd00:1122:3344:101::b]:52189","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":190,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 7789,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:25.789664667Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::b]:26231","req_id":"f53a618b-b7cf-47bc-895a-6097f7de146b","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:25.948409022Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:50950"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:25.948655968Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"uri":"/config","method":"GET","req_id":"3a6147d9-040e-49c4-8391-948b09992a4b","remote_addr":"[fd00:1122:3344:101::b]:50950","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":199,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 60632,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:28.384970739Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::a]:19995","req_id":"8e7f9a49-ba3e-4ec8-b68f-b5ea580c5424","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:28.502342847Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:47452"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:28.502583923Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"uri":"/config","method":"GET","req_id":"d8f974a1-1bce-4e48-b895-43254342e54c","remote_addr":"[fd00:1122:3344:101::a]:47452","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":179,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3892,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:28.850149351Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::c]:55063","req_id":"23f77cfb-7904-4097-86cf-57a75c64b453","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:28.986646468Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:34752"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:28.987845428Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"uri":"/config","method":"GET","req_id":"2ad1802f-6220-45c3-a7d5-882eed504100","remote_addr":"[fd00:1122:3344:101::c]:34752","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":230,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 49348,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:32.028358883Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::d]:51649","req_id":"9c5eeb70-26f7-4785-9622-131fedbfb870","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 32131,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:42.029593776Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::d]:65000","req_id":"3388a359-47cb-4d0d-a087-6a7fd470b30d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 9785,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:50.792221731Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::1b]:39660","req_id":"7b559e42-f262-414f-b3c7-8562c0a7244d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64867,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:50.804801933Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::1b]:44051","req_id":"f8980b5f-6336-43dd-84dd-923d40698e9b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53351,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:50.8097945Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::1b]:63292","req_id":"df737cec-623d-43d1-8aa3-a900bc9fe3c1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14446,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:52.030792128Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::d]:60750","req_id":"512b5299-f7a0-452f-b358-849e13cb6644","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.543453639Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:39010"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.545276906Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"new_generation":1,"req_id":"2b78b20a-bb57-4f7a-8a52-baee1bcdb20f","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.545340327Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"new_generation":1,"req_id":"2b78b20a-bb57-4f7a-8a52-baee1bcdb20f","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.545354248Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"uri":"/config","method":"PUT","req_id":"2b78b20a-bb57-4f7a-8a52-baee1bcdb20f","remote_addr":"[fd00:1122:3344:101::a]:39010","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":299,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.672900746Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:61845"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.674122617Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"new_generation":1,"req_id":"b1292f6c-fb67-4429-9b89-09be7ceae4a3","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.674154967Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"new_generation":1,"req_id":"b1292f6c-fb67-4429-9b89-09be7ceae4a3","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.674190908Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"uri":"/config","method":"PUT","req_id":"b1292f6c-fb67-4429-9b89-09be7ceae4a3","remote_addr":"[fd00:1122:3344:101::c]:61845","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":346,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.729760108Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:40700"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.731662247Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"new_generation":1,"req_id":"138cd8b5-102e-41e4-af8a-1832974c1b07","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.731722428Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"new_generation":1,"req_id":"138cd8b5-102e-41e4-af8a-1832974c1b07","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.731734848Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"uri":"/config","method":"PUT","req_id":"138cd8b5-102e-41e4-af8a-1832974c1b07","remote_addr":"[fd00:1122:3344:101::b]:40700","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":209,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 30045,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:58.690426126Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::c]:4122","req_id":"239e5fa1-0921-447a-aca2-6a9426560429","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 45442,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:58.695942567Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::b]:19334","req_id":"8fabeffc-821b-4c27-892e-a0ac3840fa1f","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:58.798016545Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:63703"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:58.798287181Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"uri":"/config","method":"GET","req_id":"972292a0-2453-4687-b5fd-9d9708d5eca0","remote_addr":"[fd00:1122:3344:101::c]:63703","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":232,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:58.829081338Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:42035"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:58.829316454Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"uri":"/config","method":"GET","req_id":"94fe7b45-bbcd-4efe-a562-552247f56231","remote_addr":"[fd00:1122:3344:101::b]:42035","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":197,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 7776,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.050077136Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::a]:61834","req_id":"7d4a5494-2037-4d4a-9767-b9e5e62172b9","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T18:00:00.144493468Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:38447"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T18:00:00.144763415Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"uri":"/config","method":"GET","req_id":"db1f36f7-9041-4e56-b009-083d1de9e528","remote_addr":"[fd00:1122:3344:101::a]:38447","local_addr":"[fd00:1122:3344:2::1]:5353","component":"http","latency_us":222,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 17964,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_382def60-3e0e-43da-b798-0e943c074eea.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_382def60-3e0e-43da-b798-0e943c074eea.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 51,\n 56,\n 50,\n 100,\n 101,\n 102,\n 54,\n 48,\n 45,\n 51,\n 101,\n 48,\n 101,\n 45,\n 52,\n 51,\n 100,\n 97,\n 45,\n 98,\n 55,\n 57,\n 56,\n 45,\n 48,\n 101,\n 57,\n 52,\n 51,\n 99,\n 48,\n 55,\n 52,\n 101,\n 101,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_382def60-3e0e-43da-b798-0e943c074eea.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.382641521Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::1b]:56479","req_id":"c68cabfe-f49e-4178-957f-b9e841bbbfa5","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 12386,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_a1cd5147-89ba-44eb-9348-15341068db10.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_a1cd5147-89ba-44eb-9348-15341068db10.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 97,\n 49,\n 99,\n 100,\n 53,\n 49,\n 52,\n 55,\n 45,\n 56,\n 57,\n 98,\n 97,\n 45,\n 52,\n 52,\n 101,\n 98,\n 45,\n 57,\n 51,\n 52,\n 56,\n 45,\n 49,\n 53,\n 51,\n 52,\n 49,\n 48,\n 54,\n 56,\n 100,\n 98,\n 49,\n 48,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_a1cd5147-89ba-44eb-9348-15341068db10.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.601978284Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::6]:49732","req_id":"1df85390-5780-4a73-8055-c9f6367c3192","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 12386,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_a1cd5147-89ba-44eb-9348-15341068db10.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_a1cd5147-89ba-44eb-9348-15341068db10.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 97,\n 49,\n 99,\n 100,\n 53,\n 49,\n 52,\n 55,\n 45,\n 56,\n 57,\n 98,\n 97,\n 45,\n 52,\n 52,\n 101,\n 98,\n 45,\n 57,\n 51,\n 52,\n 56,\n 45,\n 49,\n 53,\n 51,\n 52,\n 49,\n 48,\n 54,\n 56,\n 100,\n 98,\n 49,\n 48,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_a1cd5147-89ba-44eb-9348-15341068db10.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.602532918Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::6]:35450","req_id":"248e07ee-dc51-4baf-a720-549efedf4b0c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 24817,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 51,\n 48,\n 55,\n 102,\n 53,\n 56,\n 51,\n 52,\n 45,\n 51,\n 50,\n 51,\n 54,\n 45,\n 52,\n 48,\n 101,\n 48,\n 45,\n 56,\n 51,\n 51,\n 52,\n 45,\n 100,\n 55,\n 56,\n 57,\n 97,\n 52,\n 48,\n 48,\n 52,\n 101,\n 99,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.607401472Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::e]:62781","req_id":"dc0b83b7-0465-4a64-97bc-f391299c4a1b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 24817,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 51,\n 48,\n 55,\n 102,\n 53,\n 56,\n 51,\n 52,\n 45,\n 51,\n 50,\n 51,\n 54,\n 45,\n 52,\n 48,\n 101,\n 48,\n 45,\n 56,\n 51,\n 51,\n 52,\n 45,\n 100,\n 55,\n 56,\n 57,\n 97,\n 52,\n 48,\n 48,\n 52,\n 101,\n 99,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.607919915Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::e]:45753","req_id":"7287ddd6-bae5-47e4-abb8-038ce860086f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 13560,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_39c38aea-dee6-4629-8092-5ab4ce37b2cb.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_39c38aea-dee6-4629-8092-5ab4ce37b2cb.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 57,\n 99,\n 51,\n 56,\n 97,\n 101,\n 97,\n 45,\n 100,\n 101,\n 101,\n 54,\n 45,\n 52,\n 54,\n 50,\n 57,\n 45,\n 56,\n 48,\n 57,\n 50,\n 45,\n 53,\n 97,\n 98,\n 52,\n 99,\n 101,\n 51,\n 55,\n 98,\n 50,\n 99,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_39c38aea-dee6-4629-8092-5ab4ce37b2cb.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.65821712Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::4]:60288","req_id":"888e7b74-c19d-46c2-9395-b89476750113","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 13560,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_39c38aea-dee6-4629-8092-5ab4ce37b2cb.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_39c38aea-dee6-4629-8092-5ab4ce37b2cb.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 57,\n 99,\n 51,\n 56,\n 97,\n 101,\n 97,\n 45,\n 100,\n 101,\n 101,\n 54,\n 45,\n 52,\n 54,\n 50,\n 57,\n 45,\n 56,\n 48,\n 57,\n 50,\n 45,\n 53,\n 97,\n 98,\n 52,\n 99,\n 101,\n 51,\n 55,\n 98,\n 50,\n 99,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_39c38aea-dee6-4629-8092-5ab4ce37b2cb.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.658967229Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::4]:64841","req_id":"39eac8fb-1c17-48b0-9046-c8fa5eb19992","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43436,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_39e05c87-abdd-4311-accb-06499195f5f5.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_39e05c87-abdd-4311-accb-06499195f5f5.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 57,\n 101,\n 48,\n 53,\n 99,\n 56,\n 55,\n 45,\n 97,\n 98,\n 100,\n 100,\n 45,\n 52,\n 51,\n 49,\n 49,\n 45,\n 97,\n 99,\n 99,\n 98,\n 45,\n 48,\n 54,\n 52,\n 57,\n 57,\n 49,\n 57,\n 53,\n 102,\n 53,\n 102,\n 53,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_39e05c87-abdd-4311-accb-06499195f5f5.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.660004536Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::7]:59334","req_id":"a6bddda7-c587-42b2-be75-c85ed64a16aa","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43436,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_39e05c87-abdd-4311-accb-06499195f5f5.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_39e05c87-abdd-4311-accb-06499195f5f5.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 57,\n 101,\n 48,\n 53,\n 99,\n 56,\n 55,\n 45,\n 97,\n 98,\n 100,\n 100,\n 45,\n 52,\n 51,\n 49,\n 49,\n 45,\n 97,\n 99,\n 99,\n 98,\n 45,\n 48,\n 54,\n 52,\n 57,\n 57,\n 49,\n 57,\n 53,\n 102,\n 53,\n 102,\n 53,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_39e05c87-abdd-4311-accb-06499195f5f5.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.660537619Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::7]:40363","req_id":"65347a30-17bf-4b95-8366-c87cdd4e1e3d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1949,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_9e155ee0-5cdf-431a-b548-2719bc3491af.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_9e155ee0-5cdf-431a-b548-2719bc3491af.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 57,\n 101,\n 49,\n 53,\n 53,\n 101,\n 101,\n 48,\n 45,\n 53,\n 99,\n 100,\n 102,\n 45,\n 52,\n 51,\n 49,\n 97,\n 45,\n 98,\n 53,\n 52,\n 56,\n 45,\n 50,\n 55,\n 49,\n 57,\n 98,\n 99,\n 51,\n 52,\n 57,\n 49,\n 97,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_9e155ee0-5cdf-431a-b548-2719bc3491af.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.672155426Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::5]:54296","req_id":"caa4c165-61c8-4374-b6c7-9916817e7d18","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1949,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_9e155ee0-5cdf-431a-b548-2719bc3491af.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_9e155ee0-5cdf-431a-b548-2719bc3491af.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 57,\n 101,\n 49,\n 53,\n 53,\n 101,\n 101,\n 48,\n 45,\n 53,\n 99,\n 100,\n 102,\n 45,\n 52,\n 51,\n 49,\n 97,\n 45,\n 98,\n 53,\n 52,\n 56,\n 45,\n 50,\n 55,\n 49,\n 57,\n 98,\n 99,\n 51,\n 52,\n 57,\n 49,\n 97,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_9e155ee0-5cdf-431a-b548-2719bc3491af.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.672660739Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::5]:47225","req_id":"2df36362-2672-4acc-aafe-636775bbb0cb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23681,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.708291849Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::2]:52053","req_id":"18bb9cf3-aac4-468b-8b7c-97baf2fac5ef","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23681,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.708924095Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::2]:36275","req_id":"a7cbb02b-050e-4377-8175-ebc59cf44c27","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41484,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_70133477-80a7-453e-9b7f-ec784ac8ce06.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_70133477-80a7-453e-9b7f-ec784ac8ce06.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 55,\n 48,\n 49,\n 51,\n 51,\n 52,\n 55,\n 55,\n 45,\n 56,\n 48,\n 97,\n 55,\n 45,\n 52,\n 53,\n 51,\n 101,\n 45,\n 57,\n 98,\n 55,\n 102,\n 45,\n 101,\n 99,\n 55,\n 56,\n 52,\n 97,\n 99,\n 56,\n 99,\n 101,\n 48,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_70133477-80a7-453e-9b7f-ec784ac8ce06.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.738245854Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::3]:51141","req_id":"dfba9ced-16bc-400b-a119-8e339b2c24c0","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41484,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_70133477-80a7-453e-9b7f-ec784ac8ce06.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_70133477-80a7-453e-9b7f-ec784ac8ce06.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 55,\n 48,\n 49,\n 51,\n 51,\n 52,\n 55,\n 55,\n 45,\n 56,\n 48,\n 97,\n 55,\n 45,\n 52,\n 53,\n 51,\n 101,\n 45,\n 57,\n 98,\n 55,\n 102,\n 45,\n 101,\n 99,\n 55,\n 56,\n 52,\n 97,\n 99,\n 56,\n 99,\n 101,\n 48,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_70133477-80a7-453e-9b7f-ec784ac8ce06.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.738907641Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::3]:33327","req_id":"b6f97264-c6b4-44ac-875c-908c36369108","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 8857,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:02.031368045Z","hostname":"oxz_internal_dns_9858e302-ac4a-4b67-82d0-ad0122bd12fc","pid":5487,"peer_addr":"[fd00:1122:3344:101::d]:50407","req_id":"53c63bd6-8abf-41b2-82cf-6aeeb6bb246a","component":"dns"}