feat: improvements in package caching

asu/build.py

@@ -222,22 +222,38 @@
     job.meta["imagebuilder_status"] = "validate_manifest"
     job.save_meta()
 
+    job.meta["make_manifest_cmd"] = [
+        "make",
+        "manifest",
+        f"PROFILE={build_request.profile}",
+        f"PACKAGES={' '.join(build_cmd_packages)}",
+        "STRIP_ABI=1",
+    ]
+
     if settings.squid_cache and not is_snapshot_build(build_request.version):
         log.info("Disabling HTTPS for repositories")
         # Once APK is used for a stable release, handle `repositories`, too
         run_cmd(container, ["sed", "-i", "s|https|http|g", "repositories.conf"])
 
     returncode, job.meta["stdout"], job.meta["stderr"] = run_cmd(
-        container,
-        [
-            "make",
-            "manifest",
-            f"PROFILE={build_request.profile}",
-            f"PACKAGES={' '.join(build_cmd_packages)}",
-            "STRIP_ABI=1",
-        ],
+        container, job.meta["make_manifest_cmd"]
     )
 
+    if (
+        returncode
+        and settings.squid_cache
+        and not is_snapshot_build(build_request.version)
+    ):
+        if any(err in job.meta["stderr"] for err in ["package index are corrupt"]):
+            log.info("Retrying without proxy")
+            returncode = 0
+            run_cmd(container, ["rm", "-rf", "/builder/dl"])
+            run_cmd(container, ["sed", "-i", "s|http|https|g", "repositories.conf"])
+
+            returncode, job.meta["stdout"], job.meta["stderr"] = run_cmd(
+                container, job.meta["make_manifest_cmd"]
+            )
+
     job.save_meta()
 
     if returncode:

asu/util.py

@@ -4,6 +4,8 @@
 import json
 import logging
 import struct
+from dateutil import parser
+from datetime import datetime
 from os import getgid, getuid
 from pathlib import Path
 from re import match
@@ -434,6 +436,19 @@
     return ""
 
 
+def is_fully_compiled_version(version: str) -> bool:
+    """Check that a given version is at least 48h old"""
+    response = client_get(
+        settings.upstream_url + "/releases/" + version + "/.targets.json"
+    )
+
+    if response.extensions["from_cache"]:
+        last_modified = int(parser.parse(response.headers["Last-Modified"]).timestamp())
+        if int(datetime.now().timestamp()) - last_modified < 172800:
+            return False
+    return True
+
+
 def reload_versions(app: FastAPI) -> bool:
     response = client_get(settings.upstream_url + "/.versions.json")
 
@@ -451,6 +466,12 @@
     if response.json()["upcoming_version"]:
         app.versions.append(response.json()["upcoming_version"])
 
+    # Avoid offering versions that are not yet fully compiled
+    for label in ["stable_version", "oldstable_version", "upcoming_version"]:
+        version = response.json()[label]
+        if version and not is_fully_compiled_version(version):
+            app.versions.remove(version)
+
     for branch in settings.branches.keys():
         if branch != "SNAPSHOT":
             app.versions.append(f"{branch}-SNAPSHOT")

misc/squid.conf

@@ -1,6 +1,3 @@
-# Always revalidate cached content with the origin server
-refresh_pattern . 0 100% 0 refresh-ims
-
 # Allow caching of files up to 5GB
 maximum_object_size 5 GB
 
@@ -9,3 +6,19 @@ cache_dir ufs /var/spool/squid 50000 16 256
 
 # Set memory cache size
 cache_mem 5 GB
+
+# Consider replacing 'downloads.openwrt.org' with '.*openwrt.*' to include multiple mirrors
+# Always revalidate imagebuilders and sha256sums with the origin server
+refresh_pattern ^http://downloads.openwrt.org/releases/.*/targets/.*(imagebuilder|sha256sums).* 0 100% 0 refresh-ims
+
+# Always revalidate the content of snapshots with the origin server
+refresh_pattern -i ^http://downloads.openwrt.org/.*snapshot.* 0 100% 0 refresh-ims
+
+# Kernel modules for releases are compiled once, never expires: cache for one year
+refresh_pattern ^http://downloads.openwrt.org/releases/.*/targets/.* 525600 100% 525600 override-expire
+
+# Consider all the remaning packages as fresh for 30min (should exist a fallback to no_proxy)
+refresh_pattern ^http://downloads.openwrt.org/.* 30 100% 30 override-expire
+
+# Always revalidate cached content with the origin server
+refresh_pattern . 0 100% 0 refresh-ims