[ Apr 24 17:27:38 Enabled. ] [ Apr 24 17:27:38 Rereading configuration. ] [ Apr 24 17:27:39 Rereading configuration. ] [ Apr 24 17:27:40 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:3::1]:5353 --dns-address [fd00:1122:3344:3::1]:53 &"). ] [ Apr 24 17:27:40 Method "start" exited with status 0. ] Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } } note: configured to log to "/dev/stdout" {"msg":"config","v":0,"name":"dns-server","level":30,"time":"2026-04-24T17:27:40.774355096Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"dns_config":"Config { bind_address: [fd00:1122:3344:3::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:3::1]:5353, default_request_body_max_bytes: 104857600, default_handler_task_mode: Detached, log_headers: [] }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"} {"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2026-04-24T17:27:40.774719196Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"component":"store","path":"/data/dns"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-04-24T17:27:40.797178591Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"component":"store"} {"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2026-04-24T17:27:40.797193832Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"component":"store","keep":3} {"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2026-04-24T17:27:40.797267874Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"component":"dns","local_address":"[fd00:1122:3344:3::1]:53"} {"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2026-04-24T17:27:40.797528981Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-04-24T17:27:40.990563524Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:3::2]:33221"} {"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2026-04-24T17:27:40.991230681Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"new_generation":1,"req_id":"84191f7a-ac1c-4fc4-881a-8db1e254dd58","component":"store"} {"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2026-04-24T17:27:40.991245532Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"component":"store"} {"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2026-04-24T17:27:41.018256906Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"component":"store","keep":3} {"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2026-04-24T17:27:41.018294177Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"new_generation":1,"req_id":"84191f7a-ac1c-4fc4-881a-8db1e254dd58","component":"store"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-04-24T17:27:41.018312828Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"uri":"/config","method":"PUT","req_id":"84191f7a-ac1c-4fc4-881a-8db1e254dd58","remote_addr":"[fd00:1122:3344:3::2]:33221","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":27456,"response_code":204} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 64889,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:27:52.350859805Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::1b]:51893","req_id":"7c1b58bd-5405-4317-810c-6de6c3da6bbd","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46340,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:27:52.355706062Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::1b]:50316","req_id":"27b48e1c-94f1-40aa-8eee-ce926fea542f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 20532,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:27:52.361613785Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::1b]:35223","req_id":"b5635d23-e7dc-45b7-b5c3-01411cafd13e","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 21977,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:29:24.84664Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::d]:54178","req_id":"40f38aa8-0641-445c-9f8d-6a447bfa22dc","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 29812,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_9acc7731-ec43-4567-b154-d0a6dfca317a.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_9acc7731-ec43-4567-b154-d0a6dfca317a.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 57,\n 97,\n 99,\n 99,\n 55,\n 55,\n 51,\n 49,\n 45,\n 101,\n 99,\n 52,\n 51,\n 45,\n 52,\n 53,\n 54,\n 55,\n 45,\n 98,\n 49,\n 53,\n 52,\n 45,\n 100,\n 48,\n 97,\n 54,\n 100,\n 102,\n 99,\n 97,\n 51,\n 49,\n 55,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_9acc7731-ec43-4567-b154-d0a6dfca317a.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:29:27.61671584Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::e]:55141","req_id":"ba434b65-a1c3-4b03-8b1f-9870afae554b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 29812,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_9acc7731-ec43-4567-b154-d0a6dfca317a.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_9acc7731-ec43-4567-b154-d0a6dfca317a.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 57,\n 97,\n 99,\n 99,\n 55,\n 55,\n 51,\n 49,\n 45,\n 101,\n 99,\n 52,\n 51,\n 45,\n 52,\n 53,\n 54,\n 55,\n 45,\n 98,\n 49,\n 53,\n 52,\n 45,\n 100,\n 48,\n 97,\n 54,\n 100,\n 102,\n 99,\n 97,\n 51,\n 49,\n 55,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_9acc7731-ec43-4567-b154-d0a6dfca317a.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:29:27.618192938Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::e]:57027","req_id":"60b5a16a-4c7d-4c9b-ba33-7e98f21ae14f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 39820,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:29:28.486173242Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::1b]:50428","req_id":"1d898144-324f-45c6-8dcf-51827b7a637d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 25844,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:29:28.490744532Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::1b]:46549","req_id":"075dfd42-91aa-48e9-b6b0-c554533a3ca9","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 9038,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:29:28.494706875Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::1b]:40659","req_id":"21dd4629-1ca1-4cb9-8b64-cdc78e5b3a20","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 9435,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:29:34.820154089Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::d]:55486","req_id":"6beb7895-9f04-4e16-941d-91d813422bfa","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14894,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:29:35.109428342Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::a]:2312","req_id":"c610fa96-7360-4482-9550-f1b0ed1a729a","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 9516,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:29:35.109932496Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::c]:47909","req_id":"09e784e8-86ce-428b-b67b-12e7e819ff92","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 22386,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:29:35.325626649Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::b]:26425","req_id":"c8413986-d8ed-44c8-b138-d0f4297e9e61","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 51143,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:29:44.826453284Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::d]:62681","req_id":"76573bb0-e09c-4bc5-81ac-4b2da192fa02","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 54245,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:29:52.49658901Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::1b]:44418","req_id":"1d7a8384-594c-43b9-9bc2-554869a946da","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 26733,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: AAAA,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 28,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:29:52.504375633Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::1b]:58216","req_id":"7ef85f9d-09ad-43e4-9f44-ad241f879e41","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 36407,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"0.pool.ntp.org.\"),\n ),\n original: Query {\n name: Name(\"0.pool.ntp.org.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 1,\n 48,\n 4,\n 112,\n 111,\n 111,\n 108,\n 3,\n 110,\n 116,\n 112,\n 3,\n 111,\n 114,\n 103,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"0.pool.ntp.org.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:29:52.508959692Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::1b]:34950","req_id":"b56904eb-42c6-423f-a9cf-f36bc39756b3","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 47022,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-cluster-native._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 26,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 99,\n 108,\n 117,\n 115,\n 116,\n 101,\n 114,\n 45,\n 110,\n 97,\n 116,\n 105,\n 118,\n 101,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-cluster-native._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:29:54.827829599Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::d]:63097","req_id":"5a94cd5f-af69-4248-ba8c-750972442e3f","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52231,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_4d74a89f-e104-40b5-8baf-e3e22337547d.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_4d74a89f-e104-40b5-8baf-e3e22337547d.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 52,\n 100,\n 55,\n 52,\n 97,\n 56,\n 57,\n 102,\n 45,\n 101,\n 49,\n 48,\n 52,\n 45,\n 52,\n 48,\n 98,\n 53,\n 45,\n 56,\n 98,\n 97,\n 102,\n 45,\n 101,\n 51,\n 101,\n 50,\n 50,\n 51,\n 51,\n 55,\n 53,\n 52,\n 55,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_4d74a89f-e104-40b5-8baf-e3e22337547d.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:30:00.224366739Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::4]:46149","req_id":"e495dff4-54d4-4212-9440-50d396bfd815","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 52231,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_4d74a89f-e104-40b5-8baf-e3e22337547d.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_4d74a89f-e104-40b5-8baf-e3e22337547d.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 52,\n 100,\n 55,\n 52,\n 97,\n 56,\n 57,\n 102,\n 45,\n 101,\n 49,\n 48,\n 52,\n 45,\n 52,\n 48,\n 98,\n 53,\n 45,\n 56,\n 98,\n 97,\n 102,\n 45,\n 101,\n 51,\n 101,\n 50,\n 50,\n 51,\n 51,\n 55,\n 53,\n 52,\n 55,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_4d74a89f-e104-40b5-8baf-e3e22337547d.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:30:00.224936794Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::4]:55945","req_id":"0c09bc81-cad4-430e-91e0-31beda948889","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 6683,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_f0073fce-1841-4415-9738-0197395ec3c7.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_f0073fce-1841-4415-9738-0197395ec3c7.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 102,\n 48,\n 48,\n 55,\n 51,\n 102,\n 99,\n 101,\n 45,\n 49,\n 56,\n 52,\n 49,\n 45,\n 52,\n 52,\n 49,\n 53,\n 45,\n 57,\n 55,\n 51,\n 56,\n 45,\n 48,\n 49,\n 57,\n 55,\n 51,\n 57,\n 53,\n 101,\n 99,\n 51,\n 99,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_f0073fce-1841-4415-9738-0197395ec3c7.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:30:00.24512172Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::5]:61817","req_id":"c8c2d40d-f5f1-44a4-811c-0e2068c6275d","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 6683,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_f0073fce-1841-4415-9738-0197395ec3c7.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_f0073fce-1841-4415-9738-0197395ec3c7.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 102,\n 48,\n 48,\n 55,\n 51,\n 102,\n 99,\n 101,\n 45,\n 49,\n 56,\n 52,\n 49,\n 45,\n 52,\n 52,\n 49,\n 53,\n 45,\n 57,\n 55,\n 51,\n 56,\n 45,\n 48,\n 49,\n 57,\n 55,\n 51,\n 57,\n 53,\n 101,\n 99,\n 51,\n 99,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_f0073fce-1841-4415-9738-0197395ec3c7.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:30:00.245625293Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::5]:43806","req_id":"2f8d8841-ec4a-4082-8922-497ce444bb55","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14818,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_bd843736-59fd-4dc1-ae84-810763560162.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_bd843736-59fd-4dc1-ae84-810763560162.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 100,\n 56,\n 52,\n 51,\n 55,\n 51,\n 54,\n 45,\n 53,\n 57,\n 102,\n 100,\n 45,\n 52,\n 100,\n 99,\n 49,\n 45,\n 97,\n 101,\n 56,\n 52,\n 45,\n 56,\n 49,\n 48,\n 55,\n 54,\n 51,\n 53,\n 54,\n 48,\n 49,\n 54,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_bd843736-59fd-4dc1-ae84-810763560162.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:30:00.271630251Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::7]:64387","req_id":"ca55c642-dbf0-47f6-b270-6325088ef9ca","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 14818,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_bd843736-59fd-4dc1-ae84-810763560162.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_bd843736-59fd-4dc1-ae84-810763560162.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 98,\n 100,\n 56,\n 52,\n 51,\n 55,\n 51,\n 54,\n 45,\n 53,\n 57,\n 102,\n 100,\n 45,\n 52,\n 100,\n 99,\n 49,\n 45,\n 97,\n 101,\n 56,\n 52,\n 45,\n 56,\n 49,\n 48,\n 55,\n 54,\n 51,\n 53,\n 54,\n 48,\n 49,\n 54,\n 50,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_bd843736-59fd-4dc1-ae84-810763560162.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:30:00.272038022Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::7]:40913","req_id":"cfe95d03-6550-4ee7-be7d-f5adbe5a54d8","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 12546,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_08362669-be09-4591-b1f3-b5c2f6bcbcb7.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_08362669-be09-4591-b1f3-b5c2f6bcbcb7.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 48,\n 56,\n 51,\n 54,\n 50,\n 54,\n 54,\n 57,\n 45,\n 98,\n 101,\n 48,\n 57,\n 45,\n 52,\n 53,\n 57,\n 49,\n 45,\n 98,\n 49,\n 102,\n 51,\n 45,\n 98,\n 53,\n 99,\n 50,\n 102,\n 54,\n 98,\n 99,\n 98,\n 99,\n 98,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_08362669-be09-4591-b1f3-b5c2f6bcbcb7.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:30:00.309702744Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::6]:40989","req_id":"c4d0eda7-023a-485a-824e-a7344bb11fa1","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 12546,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_08362669-be09-4591-b1f3-b5c2f6bcbcb7.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_08362669-be09-4591-b1f3-b5c2f6bcbcb7.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 48,\n 56,\n 51,\n 54,\n 50,\n 54,\n 54,\n 57,\n 45,\n 98,\n 101,\n 48,\n 57,\n 45,\n 52,\n 53,\n 57,\n 49,\n 45,\n 98,\n 49,\n 102,\n 51,\n 45,\n 98,\n 53,\n 99,\n 50,\n 102,\n 54,\n 98,\n 99,\n 98,\n 99,\n 98,\n 55,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_08362669-be09-4591-b1f3-b5c2f6bcbcb7.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:30:00.310202207Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::6]:60488","req_id":"76c42564-a582-4362-a4aa-6b10f67ae630","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 36156,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_f320c894-7b4d-4563-84fb-ff8da32f031d.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_f320c894-7b4d-4563-84fb-ff8da32f031d.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 102,\n 51,\n 50,\n 48,\n 99,\n 56,\n 57,\n 52,\n 45,\n 55,\n 98,\n 52,\n 100,\n 45,\n 52,\n 53,\n 54,\n 51,\n 45,\n 56,\n 52,\n 102,\n 98,\n 45,\n 102,\n 102,\n 56,\n 100,\n 97,\n 51,\n 50,\n 102,\n 48,\n 51,\n 49,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_f320c894-7b4d-4563-84fb-ff8da32f031d.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:30:00.356446493Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::3]:58567","req_id":"a47f7881-570a-4b82-bb24-34e35b6d47b5","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 36156,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_cockroachdb_f320c894-7b4d-4563-84fb-ff8da32f031d.\"),\n ),\n original: Query {\n name: Name(\"oxz_cockroachdb_f320c894-7b4d-4563-84fb-ff8da32f031d.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 52,\n 111,\n 120,\n 122,\n 95,\n 99,\n 111,\n 99,\n 107,\n 114,\n 111,\n 97,\n 99,\n 104,\n 100,\n 98,\n 95,\n 102,\n 51,\n 50,\n 48,\n 99,\n 56,\n 57,\n 52,\n 45,\n 55,\n 98,\n 52,\n 100,\n 45,\n 52,\n 53,\n 54,\n 51,\n 45,\n 56,\n 52,\n 102,\n 98,\n 45,\n 102,\n 102,\n 56,\n 100,\n 97,\n 51,\n 50,\n 102,\n 48,\n 51,\n 49,\n 100,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_f320c894-7b4d-4563-84fb-ff8da32f031d.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:30:00.356951716Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::3]:52805","req_id":"ee8326e7-c675-4040-b2f7-a7886cc660ff","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 65248,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_ntp_8b295a36-c931-4d17-9e48-887898bfd60b.\"),\n ),\n original: Query {\n name: Name(\"oxz_ntp_8b295a36-c931-4d17-9e48-887898bfd60b.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 44,\n 111,\n 120,\n 122,\n 95,\n 110,\n 116,\n 112,\n 95,\n 56,\n 98,\n 50,\n 57,\n 53,\n 97,\n 51,\n 54,\n 45,\n 99,\n 57,\n 51,\n 49,\n 45,\n 52,\n 100,\n 49,\n 55,\n 45,\n 57,\n 101,\n 52,\n 56,\n 45,\n 56,\n 56,\n 55,\n 56,\n 57,\n 56,\n 98,\n 102,\n 100,\n 54,\n 48,\n 98,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_ntp_8b295a36-c931-4d17-9e48-887898bfd60b.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:30:00.455226419Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::1b]:64817","req_id":"46ebbe25-8c56-4213-b54c-bada2cb13a89","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 25953,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_9acc7731-ec43-4567-b154-d0a6dfca317a.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_9acc7731-ec43-4567-b154-d0a6dfca317a.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 57,\n 97,\n 99,\n 99,\n 55,\n 55,\n 51,\n 49,\n 45,\n 101,\n 99,\n 52,\n 51,\n 45,\n 52,\n 53,\n 54,\n 55,\n 45,\n 98,\n 49,\n 53,\n 52,\n 45,\n 100,\n 48,\n 97,\n 54,\n 100,\n 102,\n 99,\n 97,\n 51,\n 49,\n 55,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_9acc7731-ec43-4567-b154-d0a6dfca317a.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:30:00.575267589Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::e]:44137","req_id":"949dff07-b099-4599-a0f9-1ef0cde3ef72","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 25953,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_clickhouse_9acc7731-ec43-4567-b154-d0a6dfca317a.\"),\n ),\n original: Query {\n name: Name(\"oxz_clickhouse_9acc7731-ec43-4567-b154-d0a6dfca317a.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 51,\n 111,\n 120,\n 122,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 95,\n 57,\n 97,\n 99,\n 99,\n 55,\n 55,\n 51,\n 49,\n 45,\n 101,\n 99,\n 52,\n 51,\n 45,\n 52,\n 53,\n 54,\n 55,\n 45,\n 98,\n 49,\n 53,\n 52,\n 45,\n 100,\n 48,\n 97,\n 54,\n 100,\n 102,\n 99,\n 97,\n 51,\n 49,\n 55,\n 97,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_clickhouse_9acc7731-ec43-4567-b154-d0a6dfca317a.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:30:00.575883775Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::e]:57087","req_id":"5c1dd0d1-f492-401a-889e-cbffb2a7fb72","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46218,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:30:00.57878503Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::2]:60094","req_id":"98874616-13ae-49d6-bbc5-38063daec41b","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 46218,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 0,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"oxz_switch.\"),\n ),\n original: Query {\n name: Name(\"oxz_switch.\"),\n query_type: A,\n query_class: IN,\n },\n },\n ],\n original: [\n 10,\n 111,\n 120,\n 122,\n 95,\n 115,\n 119,\n 105,\n 116,\n 99,\n 104,\n 0,\n 0,\n 1,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: None,\n} SERVFAIL: server is not authoritative for name: \"oxz_switch.\"","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:30:00.579276373Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::2]:64266","req_id":"c429d4df-2463-4aff-a8a5-86654d269b05","component":"dns"} {"msg":"failed to handle incoming DNS message: MessageRequest {\n header: Header {\n id: 7906,\n message_type: Query,\n op_code: Query,\n authoritative: false,\n truncation: false,\n recursion_desired: true,\n recursion_available: false,\n authentic_data: false,\n checking_disabled: false,\n response_code: NoError,\n query_count: 1,\n answer_count: 0,\n name_server_count: 0,\n additional_count: 1,\n },\n queries: Queries {\n queries: [\n LowerQuery {\n name: LowerName(\n Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n ),\n original: Query {\n name: Name(\"_clickhouse-admin-keeper._tcp.control-plane.oxide.internal.\"),\n query_type: SRV,\n query_class: IN,\n },\n },\n ],\n original: [\n 24,\n 95,\n 99,\n 108,\n 105,\n 99,\n 107,\n 104,\n 111,\n 117,\n 115,\n 101,\n 45,\n 97,\n 100,\n 109,\n 105,\n 110,\n 45,\n 107,\n 101,\n 101,\n 112,\n 101,\n 114,\n 4,\n 95,\n 116,\n 99,\n 112,\n 13,\n 99,\n 111,\n 110,\n 116,\n 114,\n 111,\n 108,\n 45,\n 112,\n 108,\n 97,\n 110,\n 101,\n 5,\n 111,\n 120,\n 105,\n 100,\n 101,\n 8,\n 105,\n 110,\n 116,\n 101,\n 114,\n 110,\n 97,\n 108,\n 0,\n 0,\n 33,\n 0,\n 1,\n ],\n },\n answers: [],\n name_servers: [],\n additionals: [],\n sig0: [],\n edns: Some(\n Edns {\n rcode_high: 0,\n version: 0,\n flags: EdnsFlags {\n dnssec_ok: false,\n z: 0,\n },\n max_payload: 1232,\n options: OPT {\n options: [],\n },\n },\n ),\n} NXDOMAIN: _clickhouse-admin-keeper._tcp.control-plane.oxide.internal","v":0,"name":"dns-server","level":50,"time":"2026-04-24T17:30:00.89857916Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"peer_addr":"[fd00:1122:3344:101::c]:30450","req_id":"505d889b-835f-40ff-9e1e-8fbf3b5a2ef7","component":"dns"} {"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2026-04-24T17:30:01.351472889Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:101::c]:49533"} {"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2026-04-24T17:30:01.430930501Z","hostname":"oxz_internal_dns_e5f412a3-4da7-42de-81de-7bb03e00bddb","pid":5529,"uri":"/config","method":"GET","req_id":"02e72424-3dfa-44a4-b6bd-918bbe53fc7f","remote_addr":"[fd00:1122:3344:101::c]:49533","local_addr":"[fd00:1122:3344:3::1]:5353","component":"http","latency_us":52498,"response_code":200}