Honor Cache-control max-age / Expires HTTP headers during feed updates. #376

Author: lemon24

src/reader/_parser/__init__.py

@@ -35,6 +35,8 @@
 
 
 if TYPE_CHECKING:  # pragma: no cover
+    from werkzeug.datastructures import RequestCacheControl
+
     from ._lazy import Parser as Parser
 
 
@@ -213,6 +215,32 @@ def retry_after(self) -> datetime | timedelta | None:
 
         return timedelta(seconds=seconds)
 
+    @property
+    def cache_control(self) -> RequestCacheControl | None:
+        """Parsed Cache-Control header, or None if missing."""
+
+        # lazy import
+        from ._http_utils import parse_cache_control_header
+
+        value = self.headers.get('cache-control')
+        if not value:
+            return None
+
+        return parse_cache_control_header(value)
+
+    @property
+    def expires(self) -> datetime | None:
+        """Parsed Expires header, or None if missing."""
+
+        # lazy import
+        from ._http_utils import parse_date
+
+        value = self.headers.get('expires')
+        if not value:
+            return None
+
+        return parse_date(value)
+
     def get_update_after(self, now: datetime) -> datetime | None:
         """Select the best "update after" date from available headers."""
         rv = []
@@ -225,6 +253,15 @@ def get_update_after(self, now: datetime) -> datetime | None:
                     retry_after = now + retry_after
                 rv.append(retry_after)
 
+        # https://httpwg.org/specs/rfc9111.html#calculating.freshness.lifetime
+        if cache_control := self.cache_control:
+            if max_age := cache_control.max_age:
+                rv.append(now + timedelta(seconds=max_age))
+        elif expires := self.expires:
+            # TODO (#376): technically this is supposed to be against Date
+            rv.append(expires.astimezone(timezone.utc))
+        # TODO (#376): what about heuristics?
+
         return max(rv, default=None)
 
 

src/reader/_parser/_http_utils.py

@@ -4,8 +4,11 @@
 """
 
 from collections.abc import Iterable
+from functools import partial
 
 import werkzeug.http
+from werkzeug.datastructures import MIMEAccept
+from werkzeug.datastructures import ResponseCacheControl
 
 
 parse_options_header = werkzeug.http.parse_options_header
@@ -14,4 +17,9 @@
 
 
 def unparse_accept_header(values: Iterable[tuple[str, float]]) -> str:
-    return werkzeug.datastructures.MIMEAccept(values).to_header()
+    return MIMEAccept(values).to_header()
+
+
+parse_cache_control_header = partial(
+    werkzeug.http.parse_cache_control_header, cls=ResponseCacheControl
+)

src/reader/_update.py

@@ -253,6 +253,7 @@ def update(
 
         update_after = next_update_after(self.global_now, **self.config)
         if result.http_info:
+            # TODO (#376): technically this is supposed to be against request end
             http_update_after = result.http_info.get_update_after(self.global_now)
             # also accounts for it being in the past / negative
             if http_update_after and http_update_after > update_after:

tests/test_reader.py

@@ -930,10 +930,14 @@ def test_update_after_invalid_config(reader, parser, config, config_is_global):
     assert feed.update_after == datetime(2010, 1, 1, 1)
 
 
-def headers(*, ra=None):
+def headers(*, ra=None, ma=None, exp=None):
     rv = {}
     if ra is not None:
         rv['retry-after'] = ra
+    if ma is not None:
+        rv['cache-control'] = f"max-age={ma}"
+    if exp is not None:
+        rv['expires'] = exp
     return rv
 
 
@@ -979,6 +983,22 @@ def ids(value):
         (60, 429, headers(ra='Thu, 31 Dec 2009 23:40:00 GMT'), datetime(2010, 1, 1, 1)),
         (60, 429, headers(ra="not a date, not an int"), datetime(2010, 1, 1, 1)),
         (60, 200, headers(ra=6000), datetime(2010, 1, 1, 1)),
+        (60, 200, headers(ma=6000), datetime(2010, 1, 1, 2)),
+        (60, 200, headers(ma="not an int"), datetime(2010, 1, 1, 1)),
+        (
+            60,
+            200,
+            headers(exp='Fri, 01 Jan 2010 01:40:00 GMT'),
+            datetime(2010, 1, 1, 2),
+        ),
+        (
+            60,
+            200,
+            headers(ma=3000, exp='Fri, 01 Jan 2010 01:40:00 GMT'),
+            datetime(2010, 1, 1, 1),
+        ),
+        (60, 429, headers(ra=3000, ma=6000), datetime(2010, 1, 1, 2)),
+        (60, 429, headers(ra=6000, ma=3000), datetime(2010, 1, 1, 2)),
     ],
     ids=ids,
 )