readability: cover more timestamp formats

Author: irq0

src/llar/fetch/readability.clj

@@ -12,7 +12,10 @@
    [digest]
    [llar.item]
    [java-time.api :as time]
-   [clojure.spec.alpha :as s]))
+   [clojure.spec.alpha :as s])
+  (:import
+   [java.time LocalDate LocalDateTime ZoneOffset ZonedDateTime]
+   [java.time.format DateTimeFormatter DateTimeFormatterBuilder]))
 
 (defrecord ReadabilityItem
            [meta
@@ -79,6 +82,36 @@
             (assoc :body (when processed (hick-r/hickory-to-html processed)))
             (assoc :hickory processed))))))
 
+(def ^:private published-time-formatter
+  (-> (DateTimeFormatterBuilder.)
+      (.append DateTimeFormatter/ISO_LOCAL_DATE_TIME)
+      ;; Readability emits both extended (-05:00 / XXX) and compact
+      ;; (-0500 / XX) numeric offsets.
+      (.appendPattern "[XXX][XX]['['VV']']")
+      (.toFormatter)))
+
+(def ^:private local-published-time-formatter
+  (-> (DateTimeFormatterBuilder.)
+      (.append DateTimeFormatter/ISO_LOCAL_DATE)
+      ;; Upstream fixtures contain both ISO's T and a space separator.
+      (.appendPattern "['T'][' ']")
+      (.append DateTimeFormatter/ISO_LOCAL_TIME)
+      (.toFormatter)))
+
+(defn- parse-published-time [timestamp]
+  (try
+    (ZonedDateTime/parse timestamp published-time-formatter)
+    (catch java.time.format.DateTimeParseException zoned-error
+      (try
+        (.atZone (LocalDateTime/parse timestamp local-published-time-formatter)
+                 ZoneOffset/UTC)
+        (catch java.time.format.DateTimeParseException _
+          (try
+            (.atStartOfDay (LocalDate/parse timestamp DateTimeFormatter/ISO_LOCAL_DATE)
+                           ZoneOffset/UTC)
+            (catch java.time.format.DateTimeParseException _
+              (throw zoned-error))))))))
+
 (extend-protocol FetchSource
   llar.src.Readability
   (fetch-source [src _conditional-tokens]
@@ -91,10 +124,8 @@
                     (when-not (string/blank? (get-in fetch [:summary :title])) (get-in fetch [:summary :title]))
                     (when-not (string/blank? (:title metadata)) (:title metadata))
                     "")
-          pub-ts (or (when-not (string/blank? (:publishedTime data)) (time/zoned-date-time (time/formatter :iso-zoned-date-time)
-                                                                                           (:publishedTime data)))
-                     (when-not (string/blank? (:date metadata)) (time/zoned-date-time (time/formatter :iso-zoned-date-time)
-                                                                                      (:date metadata)))
+          pub-ts (or (when-not (string/blank? (:publishedTime data)) (parse-published-time (:publishedTime data)))
+                     (when-not (string/blank? (:date metadata)) (parse-published-time (:date metadata)))
                      (get-in fetch [:summary :ts]))]
       [(make-readability-item
         (fetch/make-meta src)

test/llar/fetch_test.clj

@@ -10,7 +10,7 @@
    [llar.fetch.feed]
    [llar.fetch.http]
    [llar.fetch.imap]
-   [llar.fetch.readability]
+   [llar.fetch.readability :as readability]
    [llar.converter :as converter]
    [hickory.select :as S]
    [llar.fetch.reddit :as reddit]
@@ -34,6 +34,29 @@
 
 (s/check-asserts true)
 
+(deftest readability-published-time-test
+  (let [parse-published-time @#'readability/parse-published-time]
+    (testing "compact numeric UTC offset"
+      (is (= "2016-01-07T00:00-05:00"
+             (str (parse-published-time "2016-01-07T00:00:00-0500")))))
+    (testing "ISO offset forms remain unchanged"
+      (is (= "2016-01-07T00:00Z"
+             (str (parse-published-time "2016-01-07T00:00:00Z"))))
+      (is (= "2016-01-07T00:00:00.123-05:00"
+             (str (parse-published-time "2016-01-07T00:00:00.123-05:00"))))
+      (is (= "2016-01-07T00:00-05:00[America/New_York]"
+             (str (parse-published-time
+                   "2016-01-07T00:00:00-0500[America/New_York]")))))
+    (testing "zone-less forms emitted by Readability"
+      (is (= "2018-04-05T06:00Z"
+             (str (parse-published-time "2018-04-05T06:00"))))
+      (is (= "2019-04-28T06:01:07Z"
+             (str (parse-published-time "2019-04-28 06:01:07"))))
+      (is (= "2017-11-03T03:01Z"
+             (str (parse-published-time "2017-11-03 03:01:00.000000"))))
+      (is (= "2020-09-21T00:00Z"
+             (str (parse-published-time "2020-09-21")))))))
+
 (defn fake-fetch-rss [url & _args]
   (when-not (= (str url) "http://example.com/feed.xml")
     (throw+ {:type ::test-error