FIX: rewrite Numpy 2.4 deprecation warning when loading old H5 files (fixes #1178)

by a more user-friendly warning

Author: gdementen

larray/inout/hdf.py

@@ -11,7 +11,7 @@
 from larray.core.constants import nan
 from larray.core.group import Group, LGroup, _translate_group_key_hdf
 from larray.core.metadata import Metadata
-from larray.util.misc import LHDFStore
+from larray.util.misc import LHDFStore, PickleDtypeWarningRewriter
 from larray.util.types import Scalar
 from larray.inout.session import register_file_handler
 from larray.inout.common import FileHandler, _supported_typenames, _supported_scalars_types
@@ -138,9 +138,14 @@ class PandasHDFHandler(FileHandler):
     r"""
     Handler for HDF5 files using Pandas.
     """
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.warn_catcher = None
 
     def _open_for_read(self):
         self.handle = HDFStore(self.fname, mode='r')
+        self.warn_catcher = PickleDtypeWarningRewriter(self.fname, stacklevel=6)
+        self.warn_catcher.__enter__()
 
     def _open_for_write(self):
         self.handle = HDFStore(self.fname)
@@ -197,3 +202,5 @@ def _dump_metadata(self, metadata):
 
     def close(self):
         self.handle.close()
+        if self.warn_catcher is not None:
+            self.warn_catcher.__exit__(None, None, None)

larray/tests/common.py

@@ -34,6 +34,7 @@
 
 
 NUMPY2 = NumpyVersion(np.__version__) >= '2.0.0'
+NUMPY24 = NumpyVersion(np.__version__) >= '2.4.0'
 SKIP_EXCEL_TESTS = False
 TESTDATADIR = Path(__file__).parent
 

larray/tests/test_array.py

@@ -12,7 +12,7 @@
     assert_nparray_equal, assert_nparray_nan_equal,
     needs_xlwings, needs_pytables, needs_xlsxwriter, needs_openpyxl,
     needs_matplotlib,
-    NUMPY2,
+    NUMPY2, NUMPY24,
     must_warn, must_raise
 )
 from larray import (
@@ -3847,6 +3847,16 @@ def test_hdf_roundtrip(tmp_path, meta):
     s = Session(fpath)
     assert s.names == sorted(['a0', 'a1', 'a2', 'a3', 'c0,c2', 'c0::2', 'even', ':name?with*special__[characters]'])
 
+    # test reading a file created by an older version
+    fpath = inputpath('test_session.h5')
+    msg = (f"'{fpath}' was created with an old version of NumPy. Please "
+           "rewrite the file with a recent version of NumPy to avoid future "
+           "compatibility issues.")
+    # if we do not use numpy >= 2.4, the warning is not triggered
+    num_expected = int(NUMPY24)
+    with must_warn(FutureWarning, msg=msg, num_expected=num_expected):
+        _ = read_hdf(fpath, '__groups__/a01')
+
 
 def test_from_string():
     expected = ndtest("c=c0,c1")

larray/tests/test_checked_session.py

@@ -12,7 +12,7 @@
                     ndtest, full, full_like, zeros_like, ones, ones_like, isnan)
 from larray.tests.common import (inputpath, assert_array_nan_equal, meta,                           # noqa: F401
                                  needs_pytables, needs_openpyxl, needs_xlwings,
-                                 must_warn, must_raise)
+                                 must_warn, must_raise, NUMPY24)
 from larray.tests.test_session import (a, a2, a3, anonymous, a01, ano01, b, b2, b024,               # noqa: F401
                                        c, d, e, f, g, h,
                                        assert_seq_equal, session, test_getitem, test_getattr,
@@ -155,7 +155,18 @@ class TestInheritance(CheckedSessionExample):
 
 @needs_pytables
 def test_init_checkedsession_hdf():
-    cs = CheckedSessionExample(inputpath('test_session.h5'))
+    fpath = inputpath('test_session.h5')
+    msg = (f"'{fpath}' was created with an old version of NumPy. Please "
+           "rewrite the file with a recent version of NumPy to avoid future "
+           "compatibility issues.")
+
+    # check_file is False because the warning stack level is calibrated for
+    # Session and not CheckedSession which uses a Session
+    # if we do not use numpy >= 2.4, the warning is not triggered
+    num_expected = int(NUMPY24)
+    with must_warn(FutureWarning, msg=msg, check_file=False,
+                   num_expected=num_expected):
+        cs = CheckedSessionExample(fpath)
     assert set(cs.keys()) == {'b', 'b024', 'a', 'a2', 'anonymous', 'a01', 'ano01', 'c', 'd', 'e', 'g', 'f', 'h'}
 
 

larray/tests/test_session.py

@@ -11,7 +11,8 @@
     meta, inputpath,
     assert_larray_equal, assert_array_nan_equal, assert_larray_nan_equal,
     needs_xlwings, needs_pytables, needs_openpyxl,
-    must_warn, must_raise
+    must_warn, must_raise,
+    NUMPY24
 )
 from larray.inout.common import _supported_scalars_types
 from larray import (Session, Axis, Array, Group, CheckedSession,
@@ -91,7 +92,14 @@ def test_init_session_xlsx():
 
 @needs_pytables
 def test_init_session_hdf():
-    s = Session(inputpath('test_session.h5'))
+    fpath = inputpath('test_session.h5')
+    msg = (f"'{fpath}' was created with an old version of NumPy. Please "
+           "rewrite the file with a recent version of NumPy to avoid future "
+           "compatibility issues.")
+    # if we do not use numpy >= 2.4, the warning is not triggered
+    num_expected = int(NUMPY24)
+    with must_warn(FutureWarning, msg=msg, num_expected=num_expected):
+        s = Session(fpath)
     assert list(s.keys()) == ['e', 'f', 'g', 'h', 'a', 'a2', 'anonymous', 'b', 'a01', 'ano01', 'b024']
 
 

larray/util/misc.py

@@ -16,11 +16,15 @@
 from collections import defaultdict
 
 import numpy as np
+from numpy.exceptions import VisibleDeprecationWarning
 import pandas as pd
 
 from larray.util.types import R
 from typing import Callable
 
+PICKLE_NP_DTYPE_WARNING_PATTERN = r'dtype\(\): align .*'
+PICKLE_NP_DTYPE_WARNING_RE = re.compile(PICKLE_NP_DTYPE_WARNING_PATTERN)
+
 try:
     np.set_printoptions(legacy='1.13')
 except TypeError:
@@ -799,6 +803,56 @@ def common_dtype(arrays) -> np.dtype:
     return np_array_common_dtype(arrays)
 
 
+class PickleDtypeWarningRewriter:
+    """
+    Context manager to rewrite warnings about old pickled numpy dtypes.
+
+    When reading from an HDF file, pandas may emit warnings about old pickled
+    numpy dtypes. This context manager catches these warnings and rewrites them
+    as FutureWarnings with a more user-friendly message.
+    """
+    def __init__(self, filepath, stacklevel):
+        self.filepath = filepath
+        self.stacklevel = stacklevel
+        self.warnings_catcher = None
+        self.caught_warnings = None
+
+    def __enter__(self):
+        warnings_catcher = warnings.catch_warnings(record=True)
+        self.warnings_catcher = warnings_catcher
+        self.caught_warnings = warnings_catcher.__enter__()
+        # do not use "always" to avoid showing the warning multiple times for
+        # the same file if it contains multiple pickled dtypes
+        warnings.filterwarnings('once',
+                                message=PICKLE_NP_DTYPE_WARNING_PATTERN,
+                                category=VisibleDeprecationWarning,
+                                module='pandas')
+        return None
+
+    def __exit__(self, type_, value, traceback):
+        # exit catch_warnings context manager (to restore normal showwarning)
+        self.warnings_catcher.__exit__(type_, value, traceback)
+        for warning in self.caught_warnings:
+            msg = warning.message
+            # We still need to check the warning message because other
+            # types of warnings are not (and should not be) filtered
+            if (isinstance(msg, VisibleDeprecationWarning) and
+                    PICKLE_NP_DTYPE_WARNING_RE.match(str(msg))):
+                fpath = self.filepath
+                warnings.warn(
+                    f"'{fpath}' was created with an old version of "
+                    "NumPy. Please rewrite the file with a recent "
+                    "version of NumPy to avoid future compatibility "
+                    "issues.",
+                    FutureWarning, stacklevel=self.stacklevel
+                )
+            else:
+                # Otherwise, we should re-emit the warning as-is.
+                # In practice, the stack frame of the warning is lost when
+                # we catch it, but we keep the same message and category
+                warnings.warn(warning.message, warning.category)
+
+
 class LHDFStore:
     """Context manager for pandas HDFStore."""
 
@@ -808,16 +862,26 @@ def __init__(self, filepath_or_buffer, **kwargs):
                 raise IOError('The HDFStore must be open for reading.')
             self.store = filepath_or_buffer
             self.close_store = False
+            # if the store is already open, the warning catcher should already
+            # be in place
+            warn_catcher = None
         else:
             self.store = pd.HDFStore(filepath_or_buffer, **kwargs)
             self.close_store = True
+            warn_catcher = PickleDtypeWarningRewriter(str(filepath_or_buffer),
+                                                      stacklevel=4)
+        self.warnings_catcher = warn_catcher
 
     def __enter__(self):
+        if self.warnings_catcher is not None:
+            self.warnings_catcher.__enter__()
         return self.store
 
     def __exit__(self, type_, value, traceback):
         if self.close_store:
             self.store.close()
+        if self.warnings_catcher is not None:
+            self.warnings_catcher.__exit__(type_, value, traceback)
 
 
 class SequenceZip: