diff --git a/.appveyor.yml b/.appveyor.yml index 017d855253..9adee4737b 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -3,7 +3,7 @@ environment: - TARGET_ARCH: x64 CONDA_PY: 3.10 CONDA_INSTALL_LOCN: C:\\Miniconda37-x64 - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 platform: x64 SHARED: OFF @@ -101,12 +101,12 @@ before_build: # Compiler & Generator Selection - cmd: if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2017" set OPENPMD_CMAKE_GENERATOR=Visual Studio 15 2017 - cmd: if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2019" set OPENPMD_CMAKE_GENERATOR=Visual Studio 16 2019 - - cmd: if "%TARGET_ARCH%"=="x64" set OPENPMD_CMAKE_GENERATOR=%OPENPMD_CMAKE_GENERATOR% Win64 # - cmd: if "%TARGET_ARCH%"=="x86" "C:\Program Files (x86)\Microsoft Visual Studio 15.9\VC\vcvarsall.bat" x86 # - cmd: if "%TARGET_ARCH%"=="x64" "C:\Program Files (x86)\Microsoft Visual Studio 15.9\VC\vcvarsall.bat" amd64 # CMake configure - - cmd: cmake -G "%OPENPMD_CMAKE_GENERATOR%" -DCMAKE_BUILD_TYPE=%CONFIGURATION% -DBUILD_SHARED_LIBS=%SHARED% -DBUILD_TESTING=ON -DopenPMD_USE_PYTHON=ON -DPython_EXECUTABLE="%CONDA_INSTALL_LOCN%\python.exe" -DCMAKE_INSTALL_PREFIX="%CONDA_INSTALL_LOCN%" -DCMAKE_INSTALL_BINDIR="Library\bin" ".." + - cmd: if "%TARGET_ARCH%"=="x64" cmake -G "%OPENPMD_CMAKE_GENERATOR%" -A "%TARGET_ARCH%" -DCMAKE_BUILD_TYPE=%CONFIGURATION% -DBUILD_SHARED_LIBS=%SHARED% -DBUILD_TESTING=ON -DopenPMD_USE_PYTHON=ON -DPython_EXECUTABLE="%CONDA_INSTALL_LOCN%\python.exe" -DCMAKE_INSTALL_PREFIX="%CONDA_INSTALL_LOCN%" -DCMAKE_INSTALL_BINDIR="Library\bin" ".." + - cmd: if "%TARGET_ARCH%"=="x86" cmake -G "%OPENPMD_CMAKE_GENERATOR%" -DCMAKE_BUILD_TYPE=%CONFIGURATION% -DBUILD_SHARED_LIBS=%SHARED% -DBUILD_TESTING=ON -DopenPMD_USE_PYTHON=ON -DPython_EXECUTABLE="%CONDA_INSTALL_LOCN%\python.exe" -DCMAKE_INSTALL_PREFIX="%CONDA_INSTALL_LOCN%" -DCMAKE_INSTALL_BINDIR="Library\bin" ".." build_script: - cmd: cmake --build . --config %CONFIGURATION% -j 2 diff --git a/include/openPMD/backend/PatchRecordComponent.hpp b/include/openPMD/backend/PatchRecordComponent.hpp index 63c8b34f92..fed17dfd3b 100644 --- a/include/openPMD/backend/PatchRecordComponent.hpp +++ b/include/openPMD/backend/PatchRecordComponent.hpp @@ -160,7 +160,10 @@ template inline void PatchRecordComponent::store(uint64_t idx, T data) { Datatype dtype = determineDatatype(); - if (dtype != getDatatype()) + if (dtype != getDatatype() && !isSameInteger(getDatatype()) && + !isSameFloatingPoint(getDatatype()) && + !isSameComplexFloatingPoint(getDatatype()) && + !isSameChar(getDatatype())) { std::ostringstream oss; oss << "Datatypes of patch data (" << dtype << ") and dataset (" @@ -187,7 +190,10 @@ template inline void PatchRecordComponent::store(T data) { Datatype dtype = determineDatatype(); - if (dtype != getDatatype()) + if (dtype != getDatatype() && !isSameInteger(getDatatype()) && + !isSameFloatingPoint(getDatatype()) && + !isSameComplexFloatingPoint(getDatatype()) && + !isSameChar(getDatatype())) { std::ostringstream oss; oss << "Datatypes of patch data (" << dtype << ") and dataset (" diff --git a/include/openPMD/binding/python/Numpy.hpp b/include/openPMD/binding/python/Numpy.hpp index c965f8c7de..062b76f560 100644 --- a/include/openPMD/binding/python/Numpy.hpp +++ b/include/openPMD/binding/python/Numpy.hpp @@ -32,9 +32,17 @@ namespace openPMD { inline Datatype dtype_from_numpy(pybind11::dtype const dt) { + // Use explicit PEP 3118 / struct format character codes for portable + // cross-platform behavior. This ensures consistency with + // dtype_from_bufferformat() and avoids platform-dependent interpretation + // of numpy type name strings. + // ref: https://docs.python.org/3/library/struct.html#format-characters + // ref: https://numpy.org/doc/stable/reference/arrays.interface.html // ref: https://docs.scipy.org/doc/numpy/user/basics.types.html - // ref: https://github.com/numpy/numpy/issues/10678#issuecomment-369363551 - if (dt.char_() == pybind11::dtype("b").char_()) + char const c = dt.char_(); + switch (c) + { + case 'b': // signed char if constexpr (std::is_signed_v) { return Datatype::CHAR; @@ -43,7 +51,7 @@ inline Datatype dtype_from_numpy(pybind11::dtype const dt) { return Datatype::SCHAR; } - else if (dt.char_() == pybind11::dtype("B").char_()) + case 'B': // unsigned char if constexpr (std::is_unsigned_v) { return Datatype::CHAR; @@ -52,42 +60,41 @@ inline Datatype dtype_from_numpy(pybind11::dtype const dt) { return Datatype::UCHAR; } - else if (dt.char_() == pybind11::dtype("short").char_()) + case 'h': // short return Datatype::SHORT; - else if (dt.char_() == pybind11::dtype("intc").char_()) - return Datatype::INT; - else if (dt.char_() == pybind11::dtype("int_").char_()) - return Datatype::LONG; - else if (dt.char_() == pybind11::dtype("longlong").char_()) - return Datatype::LONGLONG; - else if (dt.char_() == pybind11::dtype("ushort").char_()) + case 'H': // unsigned short return Datatype::USHORT; - else if (dt.char_() == pybind11::dtype("uintc").char_()) + case 'i': // int + return Datatype::INT; + case 'I': // unsigned int return Datatype::UINT; - else if (dt.char_() == pybind11::dtype("uint").char_()) + case 'l': // long + return Datatype::LONG; + case 'L': // unsigned long return Datatype::ULONG; - else if (dt.char_() == pybind11::dtype("ulonglong").char_()) + case 'q': // long long + return Datatype::LONGLONG; + case 'Q': // unsigned long long return Datatype::ULONGLONG; - else if (dt.char_() == pybind11::dtype("clongdouble").char_()) - return Datatype::CLONG_DOUBLE; - else if (dt.char_() == pybind11::dtype("cdouble").char_()) - return Datatype::CDOUBLE; - else if (dt.char_() == pybind11::dtype("csingle").char_()) - return Datatype::CFLOAT; - else if (dt.char_() == pybind11::dtype("longdouble").char_()) - return Datatype::LONG_DOUBLE; - else if (dt.char_() == pybind11::dtype("double").char_()) - return Datatype::DOUBLE; - else if (dt.char_() == pybind11::dtype("single").char_()) + case 'f': // float return Datatype::FLOAT; - else if (dt.char_() == pybind11::dtype("bool").char_()) + case 'd': // double + return Datatype::DOUBLE; + case 'g': // long double + return Datatype::LONG_DOUBLE; + case 'F': // complex float + return Datatype::CFLOAT; + case 'D': // complex double + return Datatype::CDOUBLE; + case 'G': // complex long double + return Datatype::CLONG_DOUBLE; + case '?': // bool return Datatype::BOOL; - else - { + default: pybind11::print(dt); throw std::runtime_error( - std::string("Datatype '") + dt.char_() + - std::string("' not known in 'dtype_from_numpy'!")); // _s.format(dt) + std::string("Datatype '") + c + + std::string("' not known in 'dtype_from_numpy'!")); } } @@ -159,6 +166,8 @@ inline Datatype dtype_from_bufferformat(std::string const &fmt) inline pybind11::dtype dtype_to_numpy(Datatype const dt) { + // Use explicit PEP 3118 format character codes for portable behavior. + // This ensures round-trip consistency with dtype_from_numpy(). using DT = Datatype; switch (dt) { @@ -177,80 +186,57 @@ inline pybind11::dtype dtype_to_numpy(Datatype const dt) case DT::SCHAR: case DT::VEC_SCHAR: return pybind11::dtype("b"); - break; case DT::UCHAR: case DT::VEC_UCHAR: return pybind11::dtype("B"); - break; - // case DT::SCHAR: - // case DT::VEC_SCHAR: - // pybind11::dtype("b"); - // break; case DT::SHORT: case DT::VEC_SHORT: - return pybind11::dtype("short"); - break; + return pybind11::dtype("h"); case DT::INT: case DT::VEC_INT: - return pybind11::dtype("intc"); - break; + return pybind11::dtype("i"); case DT::LONG: case DT::VEC_LONG: - return pybind11::dtype("int_"); - break; + return pybind11::dtype("l"); case DT::LONGLONG: case DT::VEC_LONGLONG: - return pybind11::dtype("longlong"); - break; + return pybind11::dtype("q"); case DT::USHORT: case DT::VEC_USHORT: - return pybind11::dtype("ushort"); - break; + return pybind11::dtype("H"); case DT::UINT: case DT::VEC_UINT: - return pybind11::dtype("uintc"); - break; + return pybind11::dtype("I"); case DT::ULONG: case DT::VEC_ULONG: - return pybind11::dtype("uint"); - break; + return pybind11::dtype("L"); case DT::ULONGLONG: case DT::VEC_ULONGLONG: - return pybind11::dtype("ulonglong"); - break; + return pybind11::dtype("Q"); case DT::FLOAT: case DT::VEC_FLOAT: - return pybind11::dtype("single"); - break; + return pybind11::dtype("f"); case DT::DOUBLE: case DT::VEC_DOUBLE: case DT::ARR_DBL_7: - return pybind11::dtype("double"); - break; + return pybind11::dtype("d"); case DT::LONG_DOUBLE: case DT::VEC_LONG_DOUBLE: - return pybind11::dtype("longdouble"); - break; + return pybind11::dtype("g"); case DT::CFLOAT: case DT::VEC_CFLOAT: - return pybind11::dtype("csingle"); - break; + return pybind11::dtype("F"); case DT::CDOUBLE: case DT::VEC_CDOUBLE: - return pybind11::dtype("cdouble"); - break; + return pybind11::dtype("D"); case DT::CLONG_DOUBLE: case DT::VEC_CLONG_DOUBLE: - return pybind11::dtype("clongdouble"); - break; + return pybind11::dtype("G"); case DT::BOOL: - return pybind11::dtype("bool"); // also "?" - break; + return pybind11::dtype("?"); case DT::UNDEFINED: default: - throw std::runtime_error( - "dtype_to_numpy: Invalid Datatype '{...}'!"); // _s.format(dt) - break; + throw std::runtime_error("dtype_to_numpy: Invalid Datatype!"); } } } // namespace openPMD diff --git a/share/openPMD/download_samples.ps1 b/share/openPMD/download_samples.ps1 index 9880aa891f..d479f57e75 100755 --- a/share/openPMD/download_samples.ps1 +++ b/share/openPMD/download_samples.ps1 @@ -24,10 +24,10 @@ Invoke-WebRequest https://github.com/openPMD/openPMD-example-datasets/raw/566b35 7z.exe x -r example-3d-bp4.tar 7z.exe x -r legacy_datasets.tar.gz 7z.exe x -r legacy_datasets.tar -Move-Item -Path example-3d\hdf5\* samples\git-sample\ -Move-Item -Path example-thetaMode\hdf5\* samples\git-sample\thetaMode\ -Move-Item -Path example-3d-bp4\* samples\git-sample\3d-bp4\ -Move-Item -Path legacy_datasets\* samples\git-sample\legacy\ +Move-Item -Path example-3d\hdf5\* -Destination samples\git-sample\ -Force +Move-Item -Path example-thetaMode\hdf5\* -Destination samples\git-sample\thetaMode\ -Force +Move-Item -Path example-3d-bp4\* -Destination samples\git-sample\3d-bp4\ -Force +Move-Item -Path legacy_datasets\* -Destination samples\git-sample\legacy\ -Force Remove-Item -Recurse -Force example-3d* Remove-Item -Recurse -Force example-thetaMode* Remove-Item -Recurse -Force example-3d-bp4* @@ -44,7 +44,7 @@ Remove-Item *.zip # Ref.: https://github.com/openPMD/openPMD-viewer/issues/296 Invoke-WebRequest https://github.com/openPMD/openPMD-viewer/files/5655027/diags.zip -OutFile empty_alternate_fbpic.zip Expand-Archive empty_alternate_fbpic.zip -Move-Item -Path empty_alternate_fbpic\diags\hdf5\data00000050.h5 samples\issue-sample\empty_alternate_fbpic_00000050.h5 +Move-Item -Path empty_alternate_fbpic\diags\hdf5\data00000050.h5 -Destination samples\issue-sample\empty_alternate_fbpic_00000050.h5 -Force Remove-Item -Recurse -Force empty_alternate_fbpic* cd $orgdir diff --git a/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp b/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp index c3c2fbfbd7..2b9bb02c2d 100644 --- a/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp +++ b/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp @@ -330,22 +330,13 @@ AttributeWithShapeAndResource::operator bool() const } #define OPENPMD_INSTANTIATE_GETATTRIBUTE(type) \ + template struct AttributeWithShape; \ + template struct AttributeWithShapeAndResource; \ template AttributeWithShape PreloadAdiosAttributes::getAttribute( \ std::string const &name) const; \ template auto AdiosAttributes::getAttribute( \ size_t step, adios2::IO &IO, std::string const &name) const \ - -> AttributeWithShapeAndResource; \ - template AttributeWithShapeAndResource< \ - type>::AttributeWithShapeAndResource(AttributeWithShape parent); \ - template AttributeWithShapeAndResource:: \ - AttributeWithShapeAndResource( \ - size_t len_in, \ - type const \ - *data_in, /* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \ - std::optional> resource_in); \ - template AttributeWithShapeAndResource< \ - type>::AttributeWithShapeAndResource(adios2::Attribute attr); \ - template AttributeWithShapeAndResource::operator bool() const; + -> AttributeWithShapeAndResource; ADIOS2_FOREACH_TYPE_1ARG(OPENPMD_INSTANTIATE_GETATTRIBUTE) #undef OPENPMD_INSTANTIATE_GETATTRIBUTE } // namespace openPMD::detail diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py index 500752b9b1..c22551a074 100644 --- a/test/python/unittest/API/APITest.py +++ b/test/python/unittest/API/APITest.py @@ -1692,7 +1692,7 @@ def testDataset(self): extent = [1, 1, 1] obj = io.Dataset(data_type, extent) if found_numpy: - d = np.array((1, 1, 1, ), dtype=np.int_) + d = np.array((1, 1, 1, ), dtype="l") obj2 = io.Dataset(d.dtype, d.shape) assert data_type == io.determine_datatype(d.dtype) assert obj2.dtype == obj.dtype