Merge pull request #273 from oree-xx/check_completion

Consolidate checks for completion file exists

Author: TimidRobot

scripts/2-process/gcs_process.py

@@ -317,7 +317,7 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    shared.check_for_data_files(args, FILE_PATHS, QUARTER)
+    shared.check_completion_file_exists(args, FILE_PATHS)
 
     # Count data
     file1_count = shared.path_join(PATHS["data_1-fetch"], "gcs_1_count.csv")

scripts/2-process/github_process.py

@@ -136,7 +136,7 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    shared.check_for_data_files(args, FILE_PATHS, QUARTER)
+    shared.check_completion_file_exists(args, FILE_PATHS)
     file_count = shared.path_join(PATHS["data_1-fetch"], "github_1_count.csv")
     count_data = shared.open_data_file(
         LOGGER, file_count, usecols=["TOOL_IDENTIFIER", "COUNT"]

scripts/2-process/wikipedia_process.py

@@ -158,7 +158,7 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    shared.check_for_data_files(args, FILE_PATHS, QUARTER)
+    shared.check_completion_file_exists(args, FILE_PATHS)
     file_count = shared.path_join(
         PATHS["data_1-fetch"], "wikipedia_count_by_languages.csv"
     )

scripts/3-report/gcs_report.py

@@ -36,6 +36,7 @@ def parse_arguments():
     """
     Parses command-line arguments, returns parsed arguments.
     """
+    global QUARTER
     LOGGER.info("Parsing command-line arguments")
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument(
@@ -70,28 +71,12 @@ def parse_arguments():
     if args.quarter != QUARTER:
         global PATHS
         PATHS = shared.paths_update(LOGGER, PATHS, QUARTER, args.quarter)
+        QUARTER = args.quarter
     args.logger = LOGGER
     args.paths = PATHS
     return args
 
 
-def check_report_completion(args):
-    """ "
-    The function checks for the last plot and image
-    caption created in this script. This helps to
-    immediately know if all plots in the script have
-    been created and should not be regenerated.
-
-    """
-    if args.force:
-        return
-    last_entry = shared.path_join(PATHS["data_phase"], "gcs_free_culture.png")
-    if os.path.exists(last_entry):
-        raise shared.QuantifyingException(
-            f"{last_entry} already exists. Report script completed", 0
-        )
-
-
 def gcs_intro(args):
     """
     Write Google Custom Search (GCS) introduction.
@@ -513,7 +498,8 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    check_report_completion(args)
+    last_entry = shared.path_join(PATHS["data_phase"], "gcs_free_culture.png")
+    shared.check_completion_file_exists(args, last_entry)
     gcs_intro(args)
     plot_products(args)
     plot_tool_status(args)

scripts/3-report/github_report.py

@@ -34,6 +34,7 @@ def parse_arguments():
     """
     Parses command-line arguments, returns parsed arguments.
     """
+    global QUARTER
     LOGGER.info("Parsing command-line arguments")
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument(
@@ -68,30 +69,12 @@ def parse_arguments():
     if args.quarter != QUARTER:
         global PATHS
         PATHS = shared.paths_update(LOGGER, PATHS, QUARTER, args.quarter)
+        QUARTER = args.quarter
     args.logger = LOGGER
     args.paths = PATHS
     return args
 
 
-def check_report_completion(args):
-    """ "
-    The function checks for the last plot and image
-    caption created in this script. This helps to
-    immediately know if all plots in the script have
-    been created and should not be regenerated.
-
-    """
-    if args.force:
-        return
-    last_entry = shared.path_join(
-        PATHS["data_phase"], "github_restriction.png"
-    )
-    if os.path.exists(last_entry):
-        raise shared.QuantifyingException(
-            f"{last_entry} already exists. Report script completed", 0
-        )
-
-
 def load_data(args):
     """
     Load the collected data from the CSV file.
@@ -262,7 +245,10 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    check_report_completion(args)
+    last_entry = shared.path_join(
+        PATHS["data_phase"], "github_restriction.png"
+    )
+    shared.check_completion_file_exists(args, last_entry)
     github_intro(args)
     plot_totals_by_license_type(args)
     plot_totals_by_restriction(args)

scripts/3-report/wikipedia_report.py

@@ -34,6 +34,7 @@ def parse_arguments():
     """
     Parses command-line arguments, returns parsed arguments.
     """
+    global QUARTER
     LOGGER.info("Parsing command-line arguments")
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument(
@@ -68,30 +69,12 @@ def parse_arguments():
     if args.quarter != QUARTER:
         global PATHS
         PATHS = shared.paths_update(LOGGER, PATHS, QUARTER, args.quarter)
+        QUARTER = args.quarter
     args.logger = LOGGER
     args.paths = PATHS
     return args
 
 
-def check_report_completion(args):
-    """ "
-    The function checks for the last plot and image
-    caption created in this script. This helps to
-    immediately know if all plots in the script have
-    been created and should not be regenerated.
-
-    """
-    if args.force:
-        return
-    last_entry = shared.path_join(
-        PATHS["data_phase"], "wikipedia_least_language_usage.png"
-    )
-    if os.path.exists(last_entry):
-        raise shared.QuantifyingException(
-            f"{last_entry} already exists. Report script completed", 0
-        )
-
-
 def wikipedia_intro(args):
     """
     Write Wikipedia introduction.
@@ -285,7 +268,10 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    check_report_completion(args)
+    last_entry = shared.path_join(
+        PATHS["data_phase"], "wikipedia_least_language_usage.png"
+    )
+    shared.check_completion_file_exists(args, last_entry)
     wikipedia_intro(args)
     plot_language_representation(args)
     plot_highest_language_usage(args)

scripts/shared.py

@@ -47,13 +47,21 @@ def data_to_csv(args, data, file_path):
     )
 
 
-def check_for_data_files(args, file_paths, QUARTER):
+def check_completion_file_exists(args, file_paths):
+    """ "
+    This function checks if expected output files
+    exists. If any exist and --force is not provided,
+    the script exits early by raising a QuantifyingException.
+    In the case of a report file, we check if last output exists.
+    """
     if args.force:
         return
+    if isinstance(file_paths, str):
+        file_paths = [file_paths]
     for path in file_paths:
         if os.path.exists(path):
             raise QuantifyingException(
-                f"Processed data already exists for {QUARTER}", 0
+                f"Output files already exists for {args.quarter}", 0
             )