Merge pull request #3 from juaml/enh/add_hexsha_check

refactor: use hexsha for dataset check logic

Author: synchon

changelog.d/3.added.md

@@ -0,0 +1 @@
+Add `hexsha` parameter for get operation via API and CLI and for `check_dataset`

changelog.d/3.changed.md

@@ -0,0 +1 @@
+Check commit hash instead of checking out state if dataset is installed

junifer_data/_cli.py

@@ -62,18 +62,27 @@ def cli() -> None:  # pragma: no cover
     metavar="<tag>",
     help="Tag to checkout",
 )
+@click.option(
+    "-s",
+    "--hexsha",
+    default=None,
+    type=str,
+    metavar="<hexsha>",
+    help="Commit hash to verify",
+)
 @click.option("-v", "--verbose", count=True, type=int)
 def get(
     file_path: click.Path,
     dataset_path: click.Path,
     tag: str,
+    hexsha: str,
     verbose: int,
 ) -> None:
     """Get FILE_PATH.
 
     FILE_PATH should be relative to <dataset>/<tag>, if provided.
     If not provided, <dataset> defaults to "$HOME/junifer_data/<tag>" and <tag>
-    defaults to "main".
+    defaults to "main". If <hexsha> is provided, commit hash is verified.
 
     """
     _set_log_config(verbose)
@@ -82,6 +91,7 @@ def get(
             file_path=file_path,
             dataset_path=dataset_path,
             tag=tag,
+            hexsha=hexsha,
         )
     except RuntimeError as err:
         click.echo(f"Failure: {err}", err=True)

junifer_data/_functions.py

@@ -24,6 +24,7 @@ def get(
     file_path: Path,
     dataset_path: Optional[Path] = None,
     tag: Optional[str] = None,
+    hexsha: Optional[str] = None,
 ) -> Path:
     """Fetch ``file_path`` from junifer-data dataset.
 
@@ -38,6 +39,8 @@ def get(
     tag : str or None, optional
         Tag to checkout; for example, for ``v1.0.0``, pass ``"1.0.0"``.
         If None, ``"main"`` is checked out (default None).
+    hexsha: str or None, optional
+        Commit hash to verify. If None, no verification will be performed.
 
     Returns
     -------
@@ -48,10 +51,13 @@ def get(
     ------
     RuntimeError
         If there is a problem fetching the file.
+    ValueError
+        If `hexsha` is provided but does not match the checked out tag.
+        If `hexsha` is provided for the main tag.
 
     """
     # Get dataset
-    dataset = check_dataset(data_dir=dataset_path, tag=tag)
+    dataset = check_dataset(data_dir=dataset_path, tag=tag, hexsha=hexsha)
     # Fetch file
     try:
         got = dataset.get(file_path, result_renderer="disabled")

junifer_data/_utils.py

@@ -18,9 +18,10 @@
 logger = logging.getLogger(__name__)
 
 
-def check_dataset(
+def check_dataset(  # noqa: C901
     data_dir: Union[str, Path, None] = None,
     tag: Optional[str] = None,
+    hexsha: Optional[str] = None,
 ) -> dl.Dataset:
     """Get or install junifer-data dataset.
 
@@ -33,6 +34,8 @@ def check_dataset(
     tag: str or None, optional
         Tag to checkout; for example, for ``v1.0.0``, pass ``"1.0.0"``.
         If None, ``"main"`` is checked out (default None).
+    hexsha: str or None, optional
+        Commit hash to verify. If None, no verification will be performed.
 
     Returns
     -------
@@ -43,6 +46,10 @@ def check_dataset(
     ------
     RuntimeError
         If there is a problem checking the dataset.
+    ValueError
+        If ``hexsha`` is provided for the main tag or
+        if unknown tag is provided or
+        if ``hexsha`` is provided but does not match the checked out ``tag``.
 
     """
     # Check tag
@@ -51,6 +58,10 @@ def check_dataset(
     else:
         tag = "main"
 
+    # Avoid hexsha check for main
+    if tag == "main" and hexsha is not None:
+        raise ValueError("Cannot verify hexsha for main tag.")
+
     # Set dataset location
     if data_dir is not None:
         data_dir = Path(data_dir) / tag
@@ -62,6 +73,52 @@ def check_dataset(
     if dl.Dataset(data_dir).is_installed():
         logger.debug(f"Found existing junifer-data at: {data_dir.resolve()}")
         dataset = dl.Dataset(data_dir)
+        # Check if dataset is dirty
+        if dataset.repo.dirty:
+            raise RuntimeError(
+                f"Found dirty junifer-data at: {data_dir.resolve()} . "
+                "You can clean or delete the directory."
+            )
+        if tag == "main":
+            # Main tag, use the latest commit
+            try:
+                dataset.update()
+            except CommandError as e:
+                raise RuntimeError(
+                    f"Failed to update junifer-data: {e}"
+                ) from e
+            else:
+                logger.debug("Successfully updated junifer-data")
+        else:
+            # Get commit hash for the tag
+            tag_hexsha = [
+                x["hexsha"]
+                for x in dataset.repo.get_tags()
+                if x["name"] == tag
+            ]
+            # Check for incorrect tags
+            if not tag_hexsha:
+                raise ValueError(f"Unknown tag: {tag} for junifer-data.")
+
+            # Get commit hash for HEAD
+            head_hexsha = dataset.repo.get_hexsha()
+            # Get tag hexsha
+            tag_hexsha = tag_hexsha[0]
+            # Check that it matches the expected hexsha from the tag info
+            if head_hexsha != tag_hexsha:
+                raise ValueError(
+                    f"Wrong commit checked out for tag: {tag}. "
+                    f"Expected: {tag_hexsha}, got: {head_hexsha}."
+                )
+
+            # Do hexsha verification for other tags if provided
+            # Check that the hexsha matches the expected hexsha from the user
+            # head_hexsha is now verified and can be used for checking
+            if hexsha is not None and head_hexsha != hexsha:
+                raise ValueError(
+                    f"Commit verification failed for tag: {tag}. "
+                    f"Expected: {head_hexsha}, got: {hexsha}."
+                )
     else:
         logger.debug(f"Cloning junifer-data to: {data_dir.resolve()}")
         # Clone dataset
@@ -79,23 +136,14 @@ def check_dataset(
             logger.debug(
                 f"Successfully cloned junifer-data to: {data_dir.resolve()}"
             )
-
-    # Update dataset to stay up-to-date
-    try:
-        dataset.update()
-    except CommandError as e:
-        raise RuntimeError(f"Failed to update junifer-data: {e}") from e
-    else:
-        logger.debug("Successfully updated junifer-data")
-
-    # Checkout correct state
-    try:
-        dataset.recall_state(tag)
-    except CommandError as e:
-        raise RuntimeError(
-            f"Failed to checkout state of junifer-data: {e}"
-        ) from e
-    else:
-        logger.debug("Successfully checked out state of junifer-data")
+        # Checkout correct state
+        try:
+            dataset.recall_state(tag)
+        except CommandError as e:
+            raise RuntimeError(
+                f"Failed to checkout state of junifer-data: {e}"
+            ) from e
+        else:
+            logger.debug("Successfully checked out state of junifer-data")
 
     return dataset