AP-676: do not fetch files of an unsupported mime type

Author: anarchivist

mokelumne/dags/fetch_images.py

@@ -19,6 +19,7 @@
 
 RunStatus = namedtuple('RunStatus', ('tind_id', 'status', 'path'))
 
+SUPPORTED_IMAGE_TYPES = {"image/jpeg", "image/png", "image/gif", "image/webp"}
 
 @dag(schedule=[to_process_csv], catchup=False, tags=["tind", "fetch", "batch-image"])
 def fetch_images():
@@ -58,11 +59,17 @@ def fetch_image_to_record_directory(orig_run_id: str, tind_id: str) -> RunStatus
         """Fetch an image from TIND to the target record's storage directory."""
         try:
             client = FetchTind(orig_run_id)
-            path = client.download_image_file(tind_id)
+            filemd = client.client.fetch_file_metadata(tind_id)
+            if filemd[0].get("mime") in SUPPORTED_IMAGE_TYPES:
+               path = client.download_image_file(tind_id)
+               status = "fetched"
+            else:
+                path = ""
+                status = f"skipped: Unsupported file type {filemd[0].get('mime')}"
         except Exception as ex:  # pylint: disable=broad-exception-caught
             return RunStatus(tind_id=tind_id, status=f'failed: {str(ex)}', path='')
 
-        return RunStatus(tind_id=tind_id, status='fetched', path=path)
+        return RunStatus(tind_id=tind_id, status=status, path=path)
 
     @task(outlets=[fetched_csv])
     def write_status_to_fetched_csv(orig_run_id: str, records: dict[str, list[str]],

mokelumne/dags/generate_image_descriptions.py

@@ -61,7 +61,7 @@ def read_and_batch_csv() -> list[list[dict[str, str]]]:
 
         with open(fetched_csv_path, mode="r", encoding="utf-8") as f:
             reader = csv.DictReader(f)
-            rows = list(reader)
+            rows = list(filter(lambda x: x["Status"] == "fetched", reader))
 
         # we could make the chunking a parameter or an env variable
         return [rows[i : i + 10] for i in range(0, len(rows), 10)]