refactor: define the program database in toml (#324)

* refactor: define the program database in toml

usgsprograms.txt was a fixed width comma separated file with a header row,
which could not carry a comment describing what a field means and had to be
re-aligned by hand when a value grew. The database is now usgsprograms.toml,
where each target is a table, so a field can be documented and added.

standard_switch and double_switch are standard_precision and
double_precision, since a switch does not say what it selects. The other
field names are unchanged, srcdir in particular, which is what the rest of
pymake calls a source directory.

A target name that contains a dot, such as mfnwt1.1.4, is quoted, since a
dot separates tables in toml and the target would otherwise be read as
three nested tables.

toml is read with tomllib, which is in the standard library from python
3.11, and with tomli before that.

* refactor: remove the string to boolean conversion toml does not need

* style: mark the toml reader fallback for the analysis tools

tomli is only imported on python 3.10, which the static analysis and the
coverage report do not run, so the import is marked as one they do not
report on.

* style: name the module constants in usgsprograms for what they are

program_data_file and target_keys are constants and are named for constants,
which the static analysis reported when the file the database is read from
changed.

The import of the toml reader is no longer marked, since the analysis does
not report it.

Author: jdhughes-dev

pixi.toml

@@ -29,6 +29,7 @@ pytest-xdist = "*"
 python = "3.11.*"
 requests = "*"
 ruff = "*"
+tomli = "*"
 
 [tasks]
 postinstall = "pip install --no-build-isolation --no-deps --disable-pip-version-check -e ."

pymake/pymake_build_apps.py

@@ -1,6 +1,6 @@
 """Build MODFLOW-based models and other utility software.
 
-Targets are defined in the usgsprograms database (usgsprograms.txt), which can
+Targets are defined in the usgsprograms database (usgsprograms.toml), which can
 be queried using functions in the usgsprograms module. An example of using
 :code:`pymake.build_apps()` to build MODFLOW 6 is:
 

pymake/utils/usgsprograms.py

@@ -12,12 +12,8 @@
 6. Functions to load, update, and export a USGS-style "code.json" json file
    containing information in the USGS application database
 
-A table listing the available pymake targets is included below:
-
-.. csv-table:: Available pymake targets
-   :file: ./usgsprograms.txt
-   :widths: 10, 10, 10, 20, 10, 10, 10, 10, 10
-   :header-rows: 1
+The available pymake targets are defined in ``usgsprograms.toml``, which is
+in this directory, and are listed by :code:`usgs_program_data.list_targets()`.
 
 """
 
@@ -26,6 +22,13 @@
 import os
 import sys
 import warnings
+
+# tomllib is in the standard library from python 3.11, and tomli is the same
+# reader for python 3.10, which is still supported
+try:
+    import tomllib
+except ModuleNotFoundError:
+    import tomli as tomllib
 from pathlib import Path
 
 from .download import _request_header, zip_all
@@ -77,43 +80,22 @@ class dotdict(dict):
 
 
 # data file containing the USGS program data
-program_data_file = "usgsprograms.txt"
+PROGRAM_DATA_FILE = "usgsprograms.toml"
 
 # keys to create for each target
-target_keys = (
+TARGET_KEYS = (
     "version",
     "current",
     "url",
     "dirname",
     "srcdir",
-    "standard_switch",
-    "double_switch",
+    "standard_precision",
+    "double_precision",
     "shared_object",
     "url_download_asset_date",
 )
 
 
-def _str_to_bool(s):
-    """Convert "True" and "False" strings to a boolean.
-
-    Parameters
-    ----------
-    s : str
-        String representation of boolean
-
-    Returns
-    -------
-
-    """
-    if s == "True":
-        return True
-    elif s == "False":
-        return False
-    else:
-        msg = f'Invalid string passed - "{s}"'
-        raise ValueError(msg)
-
-
 class usgs_program_data:
     """USGS program database class."""
 
@@ -130,35 +112,18 @@ def _build_usgs_database(self):
         """
         # pth = os.path.dirname(os.path.abspath(pymake.__file__))
         pth = os.path.dirname(os.path.abspath(__file__))
-        fpth = os.path.join(pth, program_data_file)
-        url_in = open(fpth, "r").read().split("\n")
+        fpth = os.path.join(pth, PROGRAM_DATA_FILE)
+        with open(fpth, "rb") as f:
+            programs = tomllib.load(f)["program"]
 
         program_data = {}
-        for line in url_in[1:]:
-            # skip blank lines
-            if len(line.strip()) < 1:
-                continue
-            # parse comma separated line
-            t = [item.strip() for item in line.split(sep=",")]
-            # programmatically build a dictionary for each target
-            d = {}
-            for idx, key in enumerate(target_keys):
-                if key in ("url_download_asset_date",):
-                    value = None
-                else:
-                    value = t[idx + 1]
-                if key in (
-                    "current",
-                    "standard_switch",
-                    "double_switch",
-                    "shared_object",
-                ):
-                    value = _str_to_bool(value)
-                d[key] = value
+        for target, entry in programs.items():
+            # programmatically build a dictionary for each target, so that a
+            # target has every key whether the file defines it or not
+            d = {key: entry.get(key) for key in TARGET_KEYS}
 
             # make it possible to access each key with a dot (.)
-            d = dotdict(d)
-            program_data[t[0]] = d
+            program_data[target] = dotdict(d)
 
         return dotdict(program_data)
 
@@ -287,9 +252,9 @@ def get_precision(key):
         """
         target = usgs_program_data().get_target(key)
         precision = []
-        if target.standard_switch:
+        if target.standard_precision:
             precision.append("default")
-        if target.double_switch:
+        if target.double_precision:
             precision.append("double")
         return precision
 
@@ -393,7 +358,7 @@ def export_json(
             sel = "the current"
         print(
             f'writing a json file ("{fpth}") of {sel} USGS programs\n'
-            f'in the "{program_data_file}" database.\n'
+            f'in the "{PROGRAM_DATA_FILE}" database.\n'
         )
         if prog_data is not None:
             for idx, key in enumerate(prog_data.keys()):
@@ -477,13 +442,13 @@ def export_json(
             for target in pop_list:
                 del prog_data[target]
 
-        # update double_switch based on executables in appdir
+        # update double_precision based on executables in appdir
         for appdir_file in appdir.iterdir():
             temp_target = appdir_file.stem
             if temp_target.endswith("dbl"):
                 temp_target = temp_target.replace("dbl", "")
                 if temp_target in prog_data.keys():
-                    prog_data[temp_target]["double_switch"] = True
+                    prog_data[temp_target]["double_precision"] = True
 
         # write code.json to root directory - used by executables CI
         with open(file_name, "w") as file_obj:
@@ -554,7 +519,7 @@ def load_json(fpth="code.json"):
             for key, value in json_dict.items():
                 try:
                     for kk in value.keys():
-                        if kk not in target_keys:
+                        if kk not in TARGET_KEYS:
                             raise KeyError(msg + f' - key ("{kk}")')
                 except:
                     raise KeyError(msg)