Merge pull request #1514 from lesserwhirls/gh-1447

Add support for blosc compression

Author: lesserwhirls

.github/workflows/test-native-compression.yml

@@ -15,24 +15,24 @@ jobs:
           ubuntu-24.04,
           ubuntu-24.04-arm,
           windows-2022,
-          macos-13,
           macos-14,
-          macos-15
+          macos-15,
+          macos-15-intel
         ]
     name: netCDF-Java Native Compression Tests
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
       - name: Setup JDK 8, 17
-        if: ${{ matrix.os == 'ubuntu-24.04' || matrix.os == 'macos-13' }}
+        if: ${{ matrix.os == 'ubuntu-24.04' || matrix.os == 'macos-15-intel' }}
         uses: actions/setup-java@v5
         with:
           distribution: 'temurin'
           java-version: |
             8
             17
       - name: Setup JDK 21
-        if: ${{ matrix.os != 'ubuntu-24.04' && matrix.os != 'macos-13' }}
+        if: ${{ matrix.os != 'ubuntu-24.04' && matrix.os != 'macos-15-intel' }}
         uses: actions/setup-java@v5
         with:
           distribution: 'temurin'
@@ -47,12 +47,12 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-gradle-
       - name: Run libaec JNA tests
-        if: ${{ matrix.os == 'ubuntu-24.04' || matrix.os == 'macos-13'  }}
+        if: ${{ matrix.os == 'ubuntu-24.04' || matrix.os == 'macos-15-intel'  }}
         run: ./gradlew -Dorg.gradle.java.installations.auto-detect=true clean :libaec-jna:test
         env:
           JDK8: /usr/thredds-test-environment/temurin8
       - name: Run libaec JNA tests (JDK 21 tests)
-        if: ${{ matrix.os != 'ubuntu-24.04' && matrix.os != 'macos-13' }}
+        if: ${{ matrix.os != 'ubuntu-24.04' && matrix.os != 'macos-15-intel' }}
         run: ./gradlew clean :libaec-jna:test21
       - uses: actions/upload-artifact@v4
         if: failure()

build-logic/src/main/kotlin/base-conventions.gradle.kts

@@ -48,6 +48,8 @@ val publicArtifacts =
     ":legacy",
     ":libaec-jna",
     ":libaec-native",
+    ":libblosc2-jna",
+    ":libblosc2-native",
     ":netcdf4",
     ":opendap",
     ":udunits",

cdm/core/src/main/java/ucar/nc2/filter/Blosc.java

@@ -1,4 +1,3 @@
-ucar.nc2.filter.Blosc$Provider
 ucar.nc2.filter.Deflate$Provider
 ucar.nc2.filter.Checksum32$Fletcher32Provider
 ucar.nc2.filter.Checksum32$Adler32Provider

cdm/core/src/main/resources/META-INF/services/ucar.nc2.filter.FilterProvider

@@ -20,6 +20,8 @@ dependencies {
   implementation(libs.jackson.databind)
   implementation(libs.slf4j.api)
 
+  runtimeOnly(project(":libblosc2-jna"))
+
   testImplementation(platform(project(":netcdf-java-testing-platform")))
 
   testImplementation(project(":cdm-s3"))
@@ -38,6 +40,8 @@ dependencies {
 
   testCompileOnly(libs.junit4)
 
+  testRuntimeOnly(project(":libblosc2-native"))
+
   testRuntimeOnly(libs.junit5.platformLauncher)
   testRuntimeOnly(libs.junit5.vintageEngine)
 }

cdm/zarr/build.gradle.kts

@@ -15,6 +15,8 @@
 import ucar.ma2.ArrayLong;
 import ucar.ma2.DataType;
 import ucar.ma2.InvalidRangeException;
+import ucar.ma2.MAMath;
+import ucar.ma2.MAMath.MinMax;
 import ucar.ma2.Section;
 import ucar.nc2.*;
 
@@ -399,4 +401,24 @@ public void testGeozarrSharedDimension() throws IOException {
       }
     }
   }
+
+  @Test
+  public void testBloscCompressedData() throws IOException {
+    NetcdfFile ncfile = NetcdfFiles.open(SCALAR_GEOZARR_DATA);
+    Variable bloscVariable = ncfile.findVariable("temperature");
+    assertThat(bloscVariable != null).isTrue();
+    Attribute compressor = bloscVariable.findAttribute("_Compressor");
+    assertThat(compressor).isNotNull();
+    assertThat(compressor.getStringValue()).isEqualTo("blosc");
+    Array data = bloscVariable.read();
+    assertThat(data).isNotNull();
+    MinMax mm = MAMath.getMinMax(data);
+    assertThat(mm).isNotNull();
+    // values from python zarr implementation
+    double expectedMin = 1.9752643660053693e-05;
+    double expectedMax = 0.9999808929333684;
+    assertThat(mm.min).isWithin(1e-9).of(expectedMin);
+    assertThat(mm.max).isWithin(1e-7).of(expectedMax);
+  }
+
 }

cdm/zarr/src/test/java/ucar/nc2/iosp/zarr/TestZarrIosp.java

@@ -64,7 +64,7 @@ sidebars:
 - netcdfJavaTutorial_sidebar
 
 # project logo (32x32), name, main landing page  (currently used in the footer)
-project_logo: https://www.unidata.ucar.edu/images/logos/thredds_netcdf-32x32.png
+project_logo: https://assets.unidata.ucar.edu/images/logos/thredds_netcdf-32x32.png
 project_name: netCDF-Java
 project_landing_page: https://unidata.ucar.edu/software/netcdf-java/
 
@@ -74,3 +74,9 @@ description: "This is a documentation site for the netCDF-Java, which is part of
 # needed for sitemap.xml file only
 # must include trailing slash, leave off the version :-)
 base_docs_url: https://docs.unidata.ucar.edu/netcdf-java/
+
+# these will appear in various doc pages
+java_version_build: 17
+java_version_runtime: 8
+libaec_version: 1.1.3.0
+cblosc2_version: 2.22.0.0

docs/src/site/_config.yml

@@ -8,8 +8,8 @@ permalink: upgrade.html
 
 ## Requirements
 
-* Java 8 or later is required to use the library.
-* Starting with version 5.10.0, Java 17 is required to build the library.
+* Java {{ site.java_version_runtime }} or later is required to use the library.
+* Java {{ site.java_version_build }} is required to build the library.
 
 ## Quick Navigation
 * [Summary of changes for v5.10.x](#netcdf-java-api-changes-510x)
@@ -29,10 +29,23 @@ permalink: upgrade.html
 Point release notes:
 * [5.10.0](https://github.com/Unidata/netcdf-java/releases/tag/v5.10.0){:target="_blank"} (_yyyy-mm-dd, unreleased_)
 
-Starting with 5.10.x, the netCDF-Java library requires Java 17 to build (although the build will produce Java 8 bytecode, so the minimum supported version is still Java 8).
+Starting with 5.10.0, the netCDF-Java library requires Java 17 to build (although the build will produce Java 8 bytecode, so the minimum supported version is still Java 8).
 Note: we are looking to update the minimum version of the JVM we support for the project.
 Please consider taking a moment to participate in the [poll on GitHub](https://github.com/Unidata/netcdf-java/discussions/1468){:target="_blank"}.
 
+The 5.10.0 release adds support for reading blosc compressed data using the C-Blosc2 native library.
+The new artifact for JNA support is `edu.ucar.unidata:libblosc2-jna` (the current version is `2.22.0.0`).
+
+### Native jar group name changes
+
+Going forard, all native jars will be published under the `edu.ucar.unidata` group.
+Their version will match the version of the native library that they contain, plus a build number.
+This is to help make clear which version of the native library is being used by the jar.
+Starting with this release, `libaec-native` will be published under the `edu.ucar.unidata` group as well.
+Consider using the netcdf-java-bom artifact to manage your netCDF-Java dependencies, as it will always contain the latest versions of the native jars.
+The new `edu.ucar.unidata:libaec-native:1.1.3.0` is equivalent to the previous `edu.ucar:libaec-native:5.9.1`.
+Please use the new group name for any `*-native` artifacts going forward.
+
 ## netCDF-Java API Changes (5.9.x)
 
 Point release notes:

docs/src/site/pages/netcdfJava/Upgrade.md

@@ -8,7 +8,7 @@ toc: false
 
 {% include image.html file="netcdf-java/tutorial/coordsystems/CoordSys.png" alt="Coordinate Systems UML" caption="Coordinate Systems UML" %}
 
-For an overview, see: [CDM Object Model]( /common_data_model_overview.html). The Coordinate Axes for a Variable must use a subset of the Variables's dimensions.
+For an overview, see: [CDM Object Model](common_data_model_overview.html). The Coordinate Axes for a Variable must use a subset of the Variables's dimensions.
 
 ## Goals of _Coordinate Attribute Convention
 

docs/src/site/pages/netcdfJava/coordattributeconvention.md

@@ -152,7 +152,7 @@ java -Xmx1g -classpath netcdfAll-<version>.jar ucar.nc2.ft.point.writer.CFPointW
 
 ## GribCdmIndex
 
-Write GRIB Collection Indexes from an XML file containing a [GRIB `<featureCollection>`](/grib_feature_collections_ref.html) XML element.
+Write GRIB Collection Indexes from an XML file containing a [GRIB `<featureCollection>`](grib_feature_collections_ref.html) XML element.
 
 ~~~bash
 java -Xmx1g -classpath netcdfAll-<version>.jar ucar.nc2.grib.collection.GribCdmIndex [options]
@@ -173,7 +173,7 @@ Example:
 java -Xmx1g -classpath netcdfAll-<version>.jar ucar.nc2.grib.collection.GribCdmIndex -fc /data/fc/gfs_example.xml
 ~~~
 
-Note that the output file is placed in the root directory of the collection, as specified by the [Collection Specification](/collection_spec_string_ref.html) of the GRIB [`<featureCollection>`](feature_collections_ref.html).
+Note that the output file is placed in the root directory of the collection, as specified by the [Collection Specification string](https://docs.unidata.ucar.edu/tds/current/userguide/collection_spec_string_ref.html){:target="_blank"} of the GRIB [`<featureCollection>`](grib_feature_collections_ref.html).
 
 ## FeatureScan