[ Mar 24 17:53:42 Enabled. ] [ Mar 24 17:53:42 Rereading configuration. ] [ Mar 24 17:53:43 Rereading configuration. ] [ Mar 24 17:53:44 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:3::1]:5353 --dns-address [fd00:1122:3344:3::1]:53 &"). ] [ Mar 24 17:53:44 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:44.380668431Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"dns_config":"Config { bind_address: [fd00:1122:3344:3::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:44.380803735Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:44.398502027Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:44.398513007Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:44.398563578Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"component":"dns","local_address":"[fd00:1122:3344:3::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:44.398928448Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:45.918728638Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:3::2]:38814"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:45.919619041Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"new_generation":1,"req_id":"48c09a06-fb87-44ae-b7c9-86e6349ea368","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:45.919640991Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:45.934996104Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:45.935019534Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"new_generation":1,"req_id":"48c09a06-fb87-44ae-b7c9-86e6349ea368","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:53:45.935030635Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"uri":"/config","method":"PUT","req_id":"48c09a06-fb87-44ae-b7c9-86e6349ea368","remote_addr":"[fd00:1122:3344:3::2]:38814","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":15832,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 59310,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:53:57.847892525Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::1b]:38804","req_id":"055c5a77-945e-42f7-97b2-e3b92511d91f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16257,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:53:57.852216196Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::1b]:39644","req_id":"93ca0a1b-f055-4ebd-8df4-344929548a5e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40028,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:53:57.857174682Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::1b]:60083","req_id":"492e29b8-abe2-4078-b8c2-d622a35d584d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 33516,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:55:58.680541033Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::1b]:62014","req_id":"54164c10-7a67-4ae8-b775-9963c4b4b301","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 29842,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:55:58.685073415Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::1b]:47862","req_id":"6a44752a-a139-41eb-8a2a-03317cd2fcd3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 11603,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:55:58.68964161Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::1b]:46556","req_id":"148fdc06-1b66-4d48-a484-a388cd2bc6e3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 34793,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:56:32.449731021Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::1b]:40120","req_id":"a1b3e07d-b567-4504-8ed1-f54b30abfd08","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 3211,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:56:32.453540132Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::1b]:46852","req_id":"cd95103e-a1e9-45a5-ba6b-2d8f18fb32d8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23891,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:56:32.457594981Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::1b]:56727","req_id":"ef1ed8bc-dd91-4394-8a50-f6041a1fa81f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 26593,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:42.049510128Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::d]:61574","req_id":"611fa033-5c9a-4d73-939d-3983661a4abd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18809,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 51,\n 48,\n 55,\n 102,\n 53,\n 56,\n 51,\n 52,\n 45,\n 51,\n 50,\n 51,\n 54,\n 45,\n 52,\n 48,\n 101,\n 48,\n 45,\n 56,\n 51,\n 51,\n 52,\n 45,\n 100,\n 55,\n 56,\n 57,\n 97,\n 52,\n 48,\n 48,\n 52,\n 101,\n 99,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:48.009004781Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::e]:39725","req_id":"342aea44-d9ed-4e6b-aa8d-7538a30ecd4f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 18809,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 51,\n 48,\n 55,\n 102,\n 53,\n 56,\n 51,\n 52,\n 45,\n 51,\n 50,\n 51,\n 54,\n 45,\n 52,\n 48,\n 101,\n 48,\n 45,\n 56,\n 51,\n 51,\n 52,\n 45,\n 100,\n 55,\n 56,\n 57,\n 97,\n 52,\n 48,\n 48,\n 52,\n 101,\n 99,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:48.010284114Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::e]:36669","req_id":"41e58f20-7252-44f2-8e01-4ba6930b45fb","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40344,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:52.022712633Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::d]:63364","req_id":"7721806d-7951-430e-94e5-679bc3775693","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 22210,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:57.70005839Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::a]:2582","req_id":"d33a0467-3237-4c2d-8647-412af1f5a948","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52046,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:57.700229914Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::c]:46948","req_id":"41aa4ea3-476c-4b99-ae4d-70e4d7cb14f6","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 17963,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:58:57.970136689Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::b]:23340","req_id":"3cc984d7-dd7b-47f6-8926-08e738731927","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 741,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:02.024272144Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::d]:60335","req_id":"4984e480-4701-4922-8252-400ea5bbbded","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 40233,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:12.025781734Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::d]:57009","req_id":"b5de74c5-099f-4a26-ba2c-c1018ac93b9b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23700,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:19.970032845Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::a]:3923","req_id":"9606f8a4-4dd1-489e-80af-c4b904fa0ede","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:20.213244428Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:40417"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:20.265683197Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"uri":"/config","method":"GET","req_id":"6596f433-e5c6-49a2-93c3-51e853c2c5c8","remote_addr":"[fd00:1122:3344:101::a]:40417","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":25726,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 24951,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:22.026933635Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::d]:54940","req_id":"048cf1ca-5748-4ea9-bbd7-32c4e9562da3","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:24.631730239Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:58013"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:24.634527541Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"new_generation":1,"req_id":"35748a98-c6f3-4a51-87f9-7909ec93eb79","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:24.634553522Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"new_generation":1,"req_id":"35748a98-c6f3-4a51-87f9-7909ec93eb79","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:24.634616133Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"uri":"/config","method":"PUT","req_id":"35748a98-c6f3-4a51-87f9-7909ec93eb79","remote_addr":"[fd00:1122:3344:101::b]:58013","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":962,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 22488,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:24.767705173Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::b]:11772","req_id":"58dc26ad-a863-4727-9518-0027de4cf628","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:24.914652186Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:34776"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:24.914942784Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"uri":"/config","method":"GET","req_id":"ba42f026-f522-4d04-865e-e4286877beb7","remote_addr":"[fd00:1122:3344:101::b]:34776","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":224,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 24964,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:25.790537669Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::b]:19927","req_id":"ae4c9276-a9b3-48ee-b85a-43b87a4cff19","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:25.947656942Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:38625"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:25.948019092Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"uri":"/config","method":"GET","req_id":"e657e03a-54bc-45f2-9b98-9480002e9f69","remote_addr":"[fd00:1122:3344:101::b]:38625","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":217,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 24962,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:28.384994259Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::a]:49533","req_id":"64770aa8-b840-4d23-a51f-de73288eb1cc","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:28.501630589Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:33747"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:28.501887735Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"uri":"/config","method":"GET","req_id":"e317c154-25b3-483b-b1ac-8702f10c0816","remote_addr":"[fd00:1122:3344:101::a]:33747","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":216,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20559,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:28.850139711Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::c]:46889","req_id":"9a2306c2-374c-48fb-b678-f68958847402","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:28.9836086Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:50484"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:28.983837906Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"uri":"/config","method":"GET","req_id":"a5862b13-448d-4ce0-9a78-4e264295f7c7","remote_addr":"[fd00:1122:3344:101::c]:50484","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":193,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 38294,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:32.028413134Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::d]:64916","req_id":"f87eea01-6071-4884-8735-2829a6b55dbf","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 36565,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:42.029626607Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::d]:61689","req_id":"c0c6fb33-dbd6-47e7-8ad5-d3a125fa5462","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 9785,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:50.792393685Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::1b]:56887","req_id":"03f31112-5e30-4248-915e-cb7c83771d90","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64867,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:50.804979587Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::1b]:62906","req_id":"741cd4a8-66c1-42a0-82da-dfcf3eecc107","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 53351,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:50.810608151Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::1b]:59247","req_id":"804d0798-7d88-4f3a-94c0-9d3d014d1ba9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 5607,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:52.030946482Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::d]:58742","req_id":"1a6ee7d8-ee78-4f80-822e-b9e20cc55296","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.542041243Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:56666"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.545371378Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"new_generation":1,"req_id":"239f1da4-af8c-4d50-bc20-cd4764e13676","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.545432919Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"new_generation":1,"req_id":"239f1da4-af8c-4d50-bc20-cd4764e13676","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.54544575Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"uri":"/config","method":"PUT","req_id":"239f1da4-af8c-4d50-bc20-cd4764e13676","remote_addr":"[fd00:1122:3344:101::a]:56666","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":342,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.672030303Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:53811"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.674091356Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"new_generation":1,"req_id":"53c12e8e-d806-47f6-8011-93f69bcf8f7d","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.674112216Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"new_generation":1,"req_id":"53c12e8e-d806-47f6-8011-93f69bcf8f7d","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.674135517Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"uri":"/config","method":"PUT","req_id":"53c12e8e-d806-47f6-8011-93f69bcf8f7d","remote_addr":"[fd00:1122:3344:101::c]:53811","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":199,"response_code":204} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.728561287Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:53383"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.731779439Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"new_generation":1,"req_id":"4b01e28e-ab6c-4896-aa96-16672a8daf56","component":"store"} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.731851921Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"new_generation":1,"req_id":"4b01e28e-ab6c-4896-aa96-16672a8daf56","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:57.731863561Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"uri":"/config","method":"PUT","req_id":"4b01e28e-ab6c-4896-aa96-16672a8daf56","remote_addr":"[fd00:1122:3344:101::b]:53383","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":317,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 349,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:58.690431586Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::c]:47425","req_id":"c8291685-8a58-4526-98dd-b626c15167ba","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 16862,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T17:59:58.695977128Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::b]:8118","req_id":"ad9847e0-cf7b-456a-a958-41bb99658a0c","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:58.797264585Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:53617"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:58.797531302Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"uri":"/config","method":"GET","req_id":"485449f5-7219-4209-a0fd-e1799bc42d54","remote_addr":"[fd00:1122:3344:101::c]:53617","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":227,"response_code":200} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:58.828399621Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::b]:53837"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T17:59:58.828635656Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"uri":"/config","method":"GET","req_id":"917e5beb-9745-4cea-90bc-b912eecc0a54","remote_addr":"[fd00:1122:3344:101::b]:53837","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":196,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 17299,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.050120427Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::a]:45721","req_id":"c151010e-e2ef-42fb-8606-fbf08be8f502","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-03-24T18:00:00.143777789Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::a]:39385"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-03-24T18:00:00.144056307Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"uri":"/config","method":"GET","req_id":"f5e2ff66-3911-4f32-bd12-a26b734e6e18","remote_addr":"[fd00:1122:3344:101::a]:39385","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":214,"response_code":200} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 17964,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_382def60-3e0e-43da-b798-0e943c074eea.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_382def60-3e0e-43da-b798-0e943c074eea.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 51,\n 56,\n 50,\n 100,\n 101,\n 102,\n 54,\n 48,\n 45,\n 51,\n 101,\n 48,\n 101,\n 45,\n 52,\n 51,\n 100,\n 97,\n 45,\n 98,\n 55,\n 57,\n 56,\n 45,\n 48,\n 101,\n 57,\n 52,\n 51,\n 99,\n 48,\n 55,\n 52,\n 101,\n 101,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_382def60-3e0e-43da-b798-0e943c074eea.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.382892947Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::1b]:62986","req_id":"c4bde92b-5ee8-45c2-8228-64ef4f148ed7","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 12386,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_a1cd5147-89ba-44eb-9348-15341068db10.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_a1cd5147-89ba-44eb-9348-15341068db10.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 97,\n 49,\n 99,\n 100,\n 53,\n 49,\n 52,\n 55,\n 45,\n 56,\n 57,\n 98,\n 97,\n 45,\n 52,\n 52,\n 101,\n 98,\n 45,\n 57,\n 51,\n 52,\n 56,\n 45,\n 49,\n 53,\n 51,\n 52,\n 49,\n 48,\n 54,\n 56,\n 100,\n 98,\n 49,\n 48,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_a1cd5147-89ba-44eb-9348-15341068db10.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.602215899Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::6]:34667","req_id":"57788d17-6212-4c33-9ecb-3ad88e5c4911","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 12386,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_a1cd5147-89ba-44eb-9348-15341068db10.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_a1cd5147-89ba-44eb-9348-15341068db10.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 97,\n 49,\n 99,\n 100,\n 53,\n 49,\n 52,\n 55,\n 45,\n 56,\n 57,\n 98,\n 97,\n 45,\n 52,\n 52,\n 101,\n 98,\n 45,\n 57,\n 51,\n 52,\n 56,\n 45,\n 49,\n 53,\n 51,\n 52,\n 49,\n 48,\n 54,\n 56,\n 100,\n 98,\n 49,\n 48,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_a1cd5147-89ba-44eb-9348-15341068db10.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.602689682Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::6]:45266","req_id":"865ce0a0-ca84-4b85-917b-4e0ecb0d8c29","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 24817,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 51,\n 48,\n 55,\n 102,\n 53,\n 56,\n 51,\n 52,\n 45,\n 51,\n 50,\n 51,\n 54,\n 45,\n 52,\n 48,\n 101,\n 48,\n 45,\n 56,\n 51,\n 51,\n 52,\n 45,\n 100,\n 55,\n 56,\n 57,\n 97,\n 52,\n 48,\n 48,\n 52,\n 101,\n 99,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.607575047Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::e]:43020","req_id":"0a9b9842-12a8-4e43-8260-675d4895438b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 24817,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 51,\n 48,\n 55,\n 102,\n 53,\n 56,\n 51,\n 52,\n 45,\n 51,\n 50,\n 51,\n 54,\n 45,\n 52,\n 48,\n 101,\n 48,\n 45,\n 56,\n 51,\n 51,\n 52,\n 45,\n 100,\n 55,\n 56,\n 57,\n 97,\n 52,\n 48,\n 48,\n 52,\n 101,\n 99,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_307f5834-3236-40e0-8334-d789a4004eca.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.608057659Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::e]:51627","req_id":"ef0f8288-f737-4e8f-904b-2be2516e6483","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 13560,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_39c38aea-dee6-4629-8092-5ab4ce37b2cb.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_39c38aea-dee6-4629-8092-5ab4ce37b2cb.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 57,\n 99,\n 51,\n 56,\n 97,\n 101,\n 97,\n 45,\n 100,\n 101,\n 101,\n 54,\n 45,\n 52,\n 54,\n 50,\n 57,\n 45,\n 56,\n 48,\n 57,\n 50,\n 45,\n 53,\n 97,\n 98,\n 52,\n 99,\n 101,\n 51,\n 55,\n 98,\n 50,\n 99,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_39c38aea-dee6-4629-8092-5ab4ce37b2cb.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.658498407Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::4]:43057","req_id":"374ded12-24fb-454e-819e-f8bafbcf5ee5","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 13560,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_39c38aea-dee6-4629-8092-5ab4ce37b2cb.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_39c38aea-dee6-4629-8092-5ab4ce37b2cb.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 57,\n 99,\n 51,\n 56,\n 97,\n 101,\n 97,\n 45,\n 100,\n 101,\n 101,\n 54,\n 45,\n 52,\n 54,\n 50,\n 57,\n 45,\n 56,\n 48,\n 57,\n 50,\n 45,\n 53,\n 97,\n 98,\n 52,\n 99,\n 101,\n 51,\n 55,\n 98,\n 50,\n 99,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_39c38aea-dee6-4629-8092-5ab4ce37b2cb.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.659160174Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::4]:52479","req_id":"0442753a-dedc-4474-9c32-a3cb72367d2c","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43436,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_39e05c87-abdd-4311-accb-06499195f5f5.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_39e05c87-abdd-4311-accb-06499195f5f5.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 57,\n 101,\n 48,\n 53,\n 99,\n 56,\n 55,\n 45,\n 97,\n 98,\n 100,\n 100,\n 45,\n 52,\n 51,\n 49,\n 49,\n 45,\n 97,\n 99,\n 99,\n 98,\n 45,\n 48,\n 54,\n 52,\n 57,\n 57,\n 49,\n 57,\n 53,\n 102,\n 53,\n 102,\n 53,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_39e05c87-abdd-4311-accb-06499195f5f5.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.66015873Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::7]:64140","req_id":"2f7a3eaa-d48f-4d3e-915e-c4abae743ae5","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 43436,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_39e05c87-abdd-4311-accb-06499195f5f5.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_39e05c87-abdd-4311-accb-06499195f5f5.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 51,\n 57,\n 101,\n 48,\n 53,\n 99,\n 56,\n 55,\n 45,\n 97,\n 98,\n 100,\n 100,\n 45,\n 52,\n 51,\n 49,\n 49,\n 45,\n 97,\n 99,\n 99,\n 98,\n 45,\n 48,\n 54,\n 52,\n 57,\n 57,\n 49,\n 57,\n 53,\n 102,\n 53,\n 102,\n 53,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_39e05c87-abdd-4311-accb-06499195f5f5.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.660721194Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::7]:53076","req_id":"76bc9ba4-3c30-4e1c-89bf-b40d6cc26e54","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1949,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_9e155ee0-5cdf-431a-b548-2719bc3491af.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_9e155ee0-5cdf-431a-b548-2719bc3491af.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 57,\n 101,\n 49,\n 53,\n 53,\n 101,\n 101,\n 48,\n 45,\n 53,\n 99,\n 100,\n 102,\n 45,\n 52,\n 51,\n 49,\n 97,\n 45,\n 98,\n 53,\n 52,\n 56,\n 45,\n 50,\n 55,\n 49,\n 57,\n 98,\n 99,\n 51,\n 52,\n 57,\n 49,\n 97,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_9e155ee0-5cdf-431a-b548-2719bc3491af.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.67231817Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::5]:59054","req_id":"90934298-51a9-4399-b776-0013baa2ab0f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 1949,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_9e155ee0-5cdf-431a-b548-2719bc3491af.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_9e155ee0-5cdf-431a-b548-2719bc3491af.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 57,\n 101,\n 49,\n 53,\n 53,\n 101,\n 101,\n 48,\n 45,\n 53,\n 99,\n 100,\n 102,\n 45,\n 52,\n 51,\n 49,\n 97,\n 45,\n 98,\n 53,\n 52,\n 56,\n 45,\n 50,\n 55,\n 49,\n 57,\n 98,\n 99,\n 51,\n 52,\n 57,\n 49,\n 97,\n 102,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_9e155ee0-5cdf-431a-b548-2719bc3491af.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.672778732Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::5]:63974","req_id":"019b8f6f-0663-4e98-832b-6cdcc8c0bee0","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23681,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.708491074Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::2]:57197","req_id":"404e1b8f-a15f-457d-a6b0-21529133a47f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 23681,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.709232403Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::2]:63487","req_id":"14785806-8c1a-47b7-b7fc-b25792db9de4","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41484,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_70133477-80a7-453e-9b7f-ec784ac8ce06.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_70133477-80a7-453e-9b7f-ec784ac8ce06.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 55,\n 48,\n 49,\n 51,\n 51,\n 52,\n 55,\n 55,\n 45,\n 56,\n 48,\n 97,\n 55,\n 45,\n 52,\n 53,\n 51,\n 101,\n 45,\n 57,\n 98,\n 55,\n 102,\n 45,\n 101,\n 99,\n 55,\n 56,\n 52,\n 97,\n 99,\n 56,\n 99,\n 101,\n 48,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_70133477-80a7-453e-9b7f-ec784ac8ce06.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.73848117Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::3]:45466","req_id":"d8416f35-8627-404f-9b10-2e7805a64917","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 41484,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_70133477-80a7-453e-9b7f-ec784ac8ce06.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_70133477-80a7-453e-9b7f-ec784ac8ce06.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 55,\n 48,\n 49,\n 51,\n 51,\n 52,\n 55,\n 55,\n 45,\n 56,\n 48,\n 97,\n 55,\n 45,\n 52,\n 53,\n 51,\n 101,\n 45,\n 57,\n 98,\n 55,\n 102,\n 45,\n 101,\n 99,\n 55,\n 56,\n 52,\n 97,\n 99,\n 56,\n 99,\n 101,\n 48,\n 54,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_70133477-80a7-453e-9b7f-ec784ac8ce06.\"","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:00.739097146Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::3]:62425","req_id":"966778d7-b840-4ae7-80d1-79b102f99bee","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 11738,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-03-24T18:00:02.031370055Z","hostname":"oxz_internal_dns_24b7985e-2941-408a-91ce-1a5c504806c4","pid":5497,"peer_addr":"[fd00:1122:3344:101::d]:55084","req_id":"e597a663-f2bc-4693-aaa1-e40ee62588cf","component":"dns"}