Merge pull request #879 from MiraGeoscience/GEOPY-2602

GEOPY-2602: Validate grid size for clipping 2D grids for a given amount of RAM

Author: domfournier

geoh5py/io/h5_reader.py

@@ -26,8 +26,10 @@
 
 import h5py
 import numpy as np
+import psutil
 
 from ..shared import FLOAT_NDV, fetch_h5_handle
+from ..shared.exceptions import MemoryValidationError
 from ..shared.utils import (
     INV_KEY_MAP,
     KEY_MAP,
@@ -38,6 +40,34 @@
 )
 
 
+def safe_load_dataset(value: h5py.Dataset, key: str, buffer: float = 0.8) -> np.ndarray:
+    """
+    Attempt to load an h5py dataset, checking memory availability first.
+
+    :param value: h5py Dataset to load.
+    :param key: Dataset key name (for error messages).
+    :param buffer: Fraction of available memory to use as a threshold (default 0.8).
+
+    :raises MemoryValidationError: If estimated memory usage exceeds threshold
+        or if loading fails due to MemoryError.
+
+    :return: Loaded numpy array or None if memory is insufficient.
+    """
+    if not 0 < buffer <= 1:
+        raise ValueError("Buffer must be between 0 and 1.")
+
+    estimated_bytes = value.size * value.dtype.itemsize
+    available_bytes = psutil.virtual_memory().available * buffer
+
+    if estimated_bytes > available_bytes:
+        raise MemoryValidationError(key, value, available_bytes)
+
+    try:
+        return value[:]
+    except MemoryError as err:
+        raise MemoryValidationError(key, value, available_bytes) from err
+
+
 class H5Reader:
     """
     Class to read information from a geoh5 file.
@@ -96,7 +126,11 @@ def fetch_attributes(
                         and isinstance(value, h5py.Dataset)
                         and value.ndim > 0
                     ):
-                        attributes[INV_KEY_MAP[key]] = value[:]
+                        attributes[INV_KEY_MAP[key]] = safe_load_dataset(
+                            value,
+                            f"{entity_type} {as_str_if_uuid(uid)} attribute '{key}'",
+                            0.8,
+                        )
 
             if "Type" in entity:
                 type_attributes = cls.fetch_type_attributes(entity["Type"])
@@ -472,7 +506,8 @@ def fetch_values(
             name = list(h5file)[0]
 
             try:
-                values = np.r_[h5file[name]["Data"][as_str_if_uuid(uid)]["Data"]]
+                dataset = h5file[name]["Data"][as_str_if_uuid(uid)]["Data"]
+                values = safe_load_dataset(dataset, f"Data {as_str_if_uuid(uid)}")
                 if isinstance(values[0], (str, bytes)):
                     values = np.asarray([as_str_if_utf8_bytes(val) for val in values])
                     if len(values) == 1:

geoh5py/shared/exceptions.py

@@ -16,7 +16,7 @@
 #  You should have received a copy of the GNU Lesser General Public License    '
 #  along with geoh5py.  If not, see <https://www.gnu.org/licenses/>.           '
 # ''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
-
+# pylint: disable=arguments-differ
 
 from __future__ import annotations
 
@@ -36,55 +36,49 @@ class Geoh5FileClosedError(ABC, Exception):
 
 
 class BaseValidationError(ABC, Exception):
-    """Base class for custom exceptions."""
+    """
+    Base class for custom exceptions.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(self.message(*args, **kwargs))
 
     @classmethod
     @abstractmethod
-    def message(cls, name, value, validation):
-        """Builds custom error message."""
-        raise NotImplementedError()
+    def message(cls, *args, **kwargs) -> str:
+        """
+        Builds custom error message.
 
+        Each subclass of BaseValidationError should implement the message class method.
+        """
 
-class JSONParameterValidationError(Exception):
-    """Error on uuid validation."""
 
-    def __init__(self, name: str, err: str):
-        super().__init__(JSONParameterValidationError.message(name, err))
+class JSONParameterValidationError(BaseValidationError):
+    """Error on uuid validation."""
 
     @classmethod
-    def message(cls, name, err):
+    def message(cls, name: str, err: str) -> str:
         return f"Malformed ui.json dictionary for parameter '{name}'. {err}"
 
 
 class OptionalValidationError(BaseValidationError):
     """Error if None value provided to non-optional parameter."""
 
-    def __init__(
-        self,
-        name: str,
-        value: Any | None,
-        validation: bool,
-    ):
-        super().__init__(OptionalValidationError.message(name, value, validation))
-
     @classmethod
-    def message(cls, name, value, validation):
+    def message(cls, name: str, *_) -> str:
         return f"Cannot set a None value to non-optional parameter: {name}."
 
 
 class AssociationValidationError(BaseValidationError):
     """Error on association between child and parent entity validation."""
 
-    def __init__(
-        self,
+    @classmethod
+    def message(
+        cls,
         name: str,
         value: Entity | PropertyGroup | UUID,
         validation: Entity | Workspace,
-    ):
-        super().__init__(AssociationValidationError.message(name, value, validation))
-
-    @classmethod
-    def message(cls, name, value, validation):
+    ) -> str:
         return (
             f"Property '{name}' with value: '{value}' must be "
             f"a child entity of parent {validation}"
@@ -94,46 +88,38 @@ def message(cls, name, value, validation):
 class PropertyGroupValidationError(BaseValidationError):
     """Error on property group validation."""
 
-    def __init__(self, name: str, value: PropertyGroup, validation: list[str]):
-        super().__init__(PropertyGroupValidationError.message(name, value, validation))
-
     @classmethod
-    def message(cls, name, value, validation):
+    def message(cls, name: str, value: PropertyGroup, validation: list[str]) -> str:
         return (
             f"Property group for '{name}' must be of type '{validation}'. "
             f"Provided '{value.name}' of type '{value.property_group_type}'"
         )
 
 
 class AtLeastOneValidationError(BaseValidationError):
-    def __init__(self, name: str, value: list[str]):
-        super().__init__(AtLeastOneValidationError.message(name, value))
+    """Error on at least one validation."""
 
     @classmethod
-    def message(cls, name, value, validation=None):
+    def message(cls, name: str, value: list[str], *_) -> str:
         opts = "'" + "', '".join(str(k) for k in value) + "'"
         return f"Must provide at least one {name}.  Options are: {opts}"
 
 
 class RequiredValidationError(BaseValidationError):
-    def __init__(self, name: str):
-        super().__init__(RequiredValidationError.message(name))
+    """Error on required parameter validation."""
 
     @classmethod
-    def message(cls, name, value=None, validation=None):
+    def message(cls, name: str, *_) -> str:
         return f"Missing required parameter: '{name}'."
 
 
 class ShapeValidationError(BaseValidationError):
     """Error on shape validation."""
 
-    def __init__(
-        self, name: str, value: tuple[int, ...], validation: tuple[int, ...] | str
-    ):
-        super().__init__(ShapeValidationError.message(name, value, validation))
-
-    @staticmethod
-    def message(name, value, validation):
+    @classmethod
+    def message(
+        cls, name: str, value: tuple[int, ...], validation: tuple[int, ...] | str
+    ) -> str:
         return (
             f"Parameter '{name}' with shape {value} was provided. "
             f"Expected {validation}."
@@ -143,11 +129,8 @@ def message(name, value, validation):
 class TypeValidationError(BaseValidationError):
     """Error on type validation."""
 
-    def __init__(self, name: str, value: str, validation: str | list[str]):
-        super().__init__(TypeValidationError.message(name, value, validation))
-
-    @staticmethod
-    def message(name, value, validation):
+    @classmethod
+    def message(cls, name: str, value: str, validation: str | list[str]) -> str:
         return f"Type '{value}' provided for '{name}' is invalid." + iterable_message(
             validation
         )
@@ -156,38 +139,50 @@ def message(name, value, validation):
 class UUIDValidationError(BaseValidationError):
     """Error on uuid string validation."""
 
-    def __init__(self, name: str, value: str):
-        super().__init__(UUIDValidationError.message(name, value))
-
-    @staticmethod
-    def message(name, value, validation=None):
+    @classmethod
+    def message(cls, name: str, value: str, *_) -> str:
         return f"Parameter '{name}' with value '{value}' is not a valid uuid string."
 
 
 class ValueValidationError(BaseValidationError):
     """Error on value validation."""
 
-    def __init__(self, name: str, value: Any, validation: list[Any]):
-        super().__init__(ValueValidationError.message(name, value, validation))
-
-    @staticmethod
-    def message(name, value, validation):
+    @classmethod
+    def message(cls, name: str, value: Any, validation: list[Any]) -> str:
         return f"Value '{value}' provided for '{name}' is invalid." + iterable_message(
             validation
         )
 
 
-def iterable_message(valid: list[Any] | None) -> str:
-    """Append possibly iterable valid: "Must be (one of): {valid}."."""
+class MemoryValidationError(BaseValidationError):
+    """Error on memory validation."""
+
+    @classmethod
+    def message(cls, name: str, value: Any, validation: float) -> str:
+        return (
+            f"Parameter '{name}' with value '{value}' "
+            f"exceeds memory limit of {validation / 1e6} MB."
+        )
+
+
+def iterable_message(valid: str | list[Any] | None) -> str:
+    """
+    Append possibly iterable valid: "Must be (one of): {valid}".
+
+    :param valid: Valid value(s) to include in message. Can be a string, list of values, or None.
+
+    :return: Message string indicating valid value(s).
+    """
+
     if valid is None:
-        msg = ""
-    elif iterable(valid, checklen=True):
+        return ""
+    if isinstance(valid, str):
+        return f" Must be: '{valid}'."
+    if iterable(valid, checklen=True):
         vstr = "'" + "', '".join(str(k) for k in valid) + "'"
-        msg = f" Must be one of: {vstr}."
-    else:
-        msg = f" Must be: '{valid[0]}'."
+        return f" Must be one of: {vstr}."
 
-    return msg
+    return f" Must be: '{valid[0]}'."
 
 
 def iterable(value: Any, checklen: bool = False) -> bool:

geoh5py/shared/utils.py

@@ -1439,8 +1439,6 @@ def enum_name_to_str(value: Any | Enum) -> Any | str:
         return [enum_name_to_str(v) for v in value]
 
     if isinstance(value, Enum):
-        if isinstance(value, str):
-            return str(value)
         return value.name.capitalize()
 
     return value

geoh5py/ui_json/ui_json.py

@@ -337,8 +337,8 @@ def to_params(
         with (
             fetch_active_workspace(workspace)
             if workspace
-            else Workspace(self.geoh5, mode="r") as geoh5
-        ):
+            else Workspace(self.geoh5, mode="r")
+        ) as geoh5:
             if geoh5 is None:
                 raise ValueError("Workspace cannot be None.")
 
@@ -471,7 +471,7 @@ def _get_dependency_links(
         For each form, there can be a group dependency ('group') to a leading
         form ('group_optional') and/or a direct dependency between forms ('dependency').
 
-        A direct dependency controls the enabled state tow ways, while the group dependency
+        A direct dependency controls the enabled state two ways, while the group dependency
         controls the enabled state only from the lead form to its dependents.
 
         :returns: Tuple of group dependencies and direct form dependencies.

poetry.lock

@@ -67,6 +67,7 @@ h5py = "^3.15.0"
 numpy = "~2.4.0"
 Pillow = "~12.1.0"
 pydantic = "~2.12.0"
+psutil = "^7.2.2"
 
 [tool.poetry.group.dev.dependencies]
 lockfile = "^0.12.2"