feat: updated coredns compatibility and added scraper

static/compatibilities.yaml

@@ -4401,81 +4401,112 @@ addons:
   release_url: https://github.com/coredns/coredns/releases/tag/{vsn}
   helm_repository_url: https://coredns.github.io/helm
   versions:
+  - version: 1.12.0
+    kube: ['1.33']
+    requirements: []
+    incompatibilities: []
+    summary: null
+    chart_version: 1.42.2
+    images: ['coredns/coredns:1.12.0']
   - version: 1.11.3
     kube: ['1.31']
     requirements: []
     incompatibilities: []
-    chart_version: 1.32.0
+    summary: null
+    chart_version: 1.36.1
+    images: ['coredns/coredns:1.11.3']
   - version: 1.11.1
     kube: ['1.29']
     requirements: []
     incompatibilities: []
-    chart_version: 1.26.0
+    summary: null
+    chart_version: 1.31.0
+    images: ['coredns/coredns:1.11.1']
   - version: 1.10.1
     kube: ['1.27']
     requirements: []
     incompatibilities: []
-    chart_version: 1.21.0
+    summary: null
+    chart_version: 1.24.5
+    images: ['coredns/coredns:1.10.1']
   - version: 1.9.3
     kube: ['1.25']
     requirements: []
     incompatibilities: []
-    chart_version: 1.19.4
+    summary: null
+    chart_version: 1.19.6
+    images: ['coredns/coredns:1.9.3']
   - version: 1.8.6
     kube: ['1.23']
     requirements: []
     incompatibilities: []
-    chart_version: 1.16.5
+    summary: null
+    chart_version: 1.16.6
+    images: ['coredns/coredns:1.8.6']
   - version: 1.8.4
     kube: ['1.22']
     requirements: []
     incompatibilities: []
-    chart_version: 1.16.0
+    summary: null
+    chart_version: 1.16.4
+    images: ['coredns/coredns:1.8.4']
   - version: 1.8.0
     kube: ['1.21']
     requirements: []
     incompatibilities: []
-    chart_version: 1.14.0
+    summary: null
+    chart_version: 1.15.1
+    images: ['coredns/coredns:1.8.0']
   - version: 1.7.0
     kube: ['1.19']
     requirements: []
     incompatibilities: []
+    summary: null
   - version: 1.6.7
     kube: ['1.18']
     requirements: []
     incompatibilities: []
+    summary: null
   - version: 1.6.5
     kube: ['1.17']
     requirements: []
     incompatibilities: []
+    summary: null
   - version: 1.6.2
     kube: ['1.16']
     requirements: []
     incompatibilities: []
+    summary: null
   - version: 1.3.1
     kube: ['1.15', '1.14']
     requirements: []
     incompatibilities: []
+    summary: null
   - version: 1.2.6
     kube: ['1.13']
     requirements: []
     incompatibilities: []
+    summary: null
   - version: 1.2.2
     kube: ['1.12']
     requirements: []
     incompatibilities: []
+    summary: null
   - version: 1.1.3
     kube: ['1.11']
     requirements: []
     incompatibilities: []
+    summary: null
   - version: 1.0.6
     kube: ['1.10']
     requirements: []
     incompatibilities: []
+    summary: null
   - version: 1.0.1
     kube: ['1.09']
     requirements: []
     incompatibilities: []
+    summary: null
   name: coredns
 - icon: https://avatars.githubusercontent.com/u/112438027?s=200&v=4
   git_url: https://github.com/cloudnative-pg/cloudnative-pg

static/compatibilities/coredns.yaml

@@ -2,78 +2,109 @@ icon: https://avatars.githubusercontent.com/u/21110084?s=200&v=4
 release_url: https://github.com/coredns/coredns/releases/tag/{vsn}
 helm_repository_url: https://coredns.github.io/helm
 versions:
+- version: 1.12.0
+  kube: ['1.33']
+  requirements: []
+  incompatibilities: []
+  summary: null
+  chart_version: 1.42.2
+  images: ['coredns/coredns:1.12.0']
 - version: 1.11.3
   kube: ['1.31']
   requirements: []
   incompatibilities: []
-  chart_version: 1.32.0
+  summary: null
+  chart_version: 1.36.1
+  images: ['coredns/coredns:1.11.3']
 - version: 1.11.1
   kube: ['1.29']
   requirements: []
   incompatibilities: []
-  chart_version: 1.26.0
+  summary: null
+  chart_version: 1.31.0
+  images: ['coredns/coredns:1.11.1']
 - version: 1.10.1
   kube: ['1.27']
   requirements: []
   incompatibilities: []
-  chart_version: 1.21.0
+  summary: null
+  chart_version: 1.24.5
+  images: ['coredns/coredns:1.10.1']
 - version: 1.9.3
   kube: ['1.25']
   requirements: []
   incompatibilities: []
-  chart_version: 1.19.4
+  summary: null
+  chart_version: 1.19.6
+  images: ['coredns/coredns:1.9.3']
 - version: 1.8.6
   kube: ['1.23']
   requirements: []
   incompatibilities: []
-  chart_version: 1.16.5
+  summary: null
+  chart_version: 1.16.6
+  images: ['coredns/coredns:1.8.6']
 - version: 1.8.4
   kube: ['1.22']
   requirements: []
   incompatibilities: []
-  chart_version: 1.16.0
+  summary: null
+  chart_version: 1.16.4
+  images: ['coredns/coredns:1.8.4']
 - version: 1.8.0
   kube: ['1.21']
   requirements: []
   incompatibilities: []
-  chart_version: 1.14.0
+  summary: null
+  chart_version: 1.15.1
+  images: ['coredns/coredns:1.8.0']
 - version: 1.7.0
   kube: ['1.19']
   requirements: []
   incompatibilities: []
+  summary: null
 - version: 1.6.7
   kube: ['1.18']
   requirements: []
   incompatibilities: []
+  summary: null
 - version: 1.6.5
   kube: ['1.17']
   requirements: []
   incompatibilities: []
+  summary: null
 - version: 1.6.2
   kube: ['1.16']
   requirements: []
   incompatibilities: []
+  summary: null
 - version: 1.3.1
   kube: ['1.15', '1.14']
   requirements: []
   incompatibilities: []
+  summary: null
 - version: 1.2.6
   kube: ['1.13']
   requirements: []
   incompatibilities: []
+  summary: null
 - version: 1.2.2
   kube: ['1.12']
   requirements: []
   incompatibilities: []
+  summary: null
 - version: 1.1.3
   kube: ['1.11']
   requirements: []
   incompatibilities: []
+  summary: null
 - version: 1.0.6
   kube: ['1.10']
   requirements: []
   incompatibilities: []
+  summary: null
 - version: 1.0.1
   kube: ['1.09']
   requirements: []
   incompatibilities: []
+  summary: null

utils/compatibility/scrapers/coredns.py

@@ -1,4 +1,3 @@
-from bs4 import BeautifulSoup
 from utils import (
     print_error,
     fetch_page,
@@ -9,22 +8,34 @@
 import re
 
 app_name = "coredns"
-compatibility_url = "https://github.com/coredns/deployment/blob/master/kubernetes/CoreDNS-k8s_version.md"  # noqa
+compatibility_url = (
+    "https://raw.githubusercontent.com/coredns/deployment/master/kubernetes/CoreDNS-k8s_version.md"
+)
 
 
-def parse_page(content):
-    soup = BeautifulSoup(content, "html.parser")
-    table = soup.find("table")
-    return table
+def parse_markdown_table(markdown: str) -> list[tuple[str, str]]:
+    rows: list[tuple[str, str]] = []
+    for line in markdown.splitlines():
+        line = line.strip()
+        if not line.startswith("|"):
+            continue
+        cols = [c.strip() for c in line.strip("|").split("|")]
+        if len(cols) < 2:
+            continue
+        header = cols[0].lower()
+        if header.startswith("kubernetes version"):
+            continue
+        if re.fullmatch(r":?-+:?", cols[0]):
+            continue
+        rows.append((cols[0], cols[1]))
+    return rows
 
 
-def extract_table_data(table, chart_versions):
-    rows = []
-    table_rows = table.find_all("tr")
-    for row in table_rows[1:]:  # Skip header row
-        cols = row.find_all("td")
-        kubernetes_versions = cols[0].text.strip()
-        coredns_version = cols[1].text.strip()
+def extract_table_data(table_rows: list[tuple[str, str]], chart_versions):
+    rows: list[OrderedDict] = []
+    for kube_cell, coredns_cell in table_rows:  # Skip header row
+        kubernetes_versions = kube_cell.strip()
+        coredns_version = coredns_cell.strip()
 
         # Remove leading 'v' from CoreDNS version
         coredns_match = re.search(r"\d+\.\d+\.\d+", coredns_version)
@@ -46,15 +57,15 @@ def extract_table_data(table, chart_versions):
             continue
 
         rows.append(
-            rows.append(
-                OrderedDict([
+            OrderedDict(
+                [
                     ("version", coredns_version),
                     ("kube", kubernetes_versions_list),
                     ("chart_version", chart_version),
                     ("images", []),
                     ("requirements", []),
                     ("incompatibilities", []),
-                ])
+                ]
             )
         )
     return rows
@@ -66,18 +77,18 @@ def scrape():
         print_error("Failed to fetch page content.")
         return
 
-    tables = parse_page(page_content)
-    if not tables:
+    markdown = page_content.decode("utf-8", errors="replace")
+    table_rows = parse_markdown_table(markdown)
+    if not table_rows:
         print_error("No tables found in the page content.")
         return
 
     chart_versions = get_chart_versions(app_name)
-    rows = extract_table_data(tables, chart_versions)
+    rows = extract_table_data(table_rows, chart_versions)
     if not rows:
         print_error("No compatibility information found.")
         return
 
     update_compatibility_info(
         f"../../static/compatibilities/{app_name}.yaml", rows
     )
-