geomesa/.github/workflows/build-and-test.yml at main · locationtech/geomesa · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
name: build

on:
  push:
    branches:
      - main
      - '[0-9]+.[0-9]+.x'
    tags:
      - '**'
  pull_request:

permissions:  # added using https://github.com/step-security/secure-repo
  contents: read

env:
  JAVA_VERSION: 17
  MAVEN_CLI_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false --batch-mode -Dlicense.skip=true
  MAVEN_COMPILE_ARGS: clean install -Dmaven.test.skip -Dmaven.assembly.skip=true -Dmaven.source.skip -Pskip-spark-runtimes
  MAVEN_TEST_ARGS: test -Dmaven.main.skip

jobs:
  build:
    name: test / ${{ matrix.projects.name }} / ${{ matrix.scala-version }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        scala-version: [ "2.12", "2.13" ]
        projects:
          - name: accumulo-ds-1
            list: geomesa-accumulo/geomesa-accumulo-datastore -Dtest="AccumuloDataStoreTest,AccumuloDataStoreQueryTest,AccumuloDataStoreStatsTest,AttributeIndexStrategyTest,AccumuloDataStoreAlterSchemaTest,ArrowBatchIteratorTest,MultiIteratorTest,AccumuloDataStoreTransformsTest,ArrowDictionaryTest,VisibilitiesTest"
          - name: accumulo-ds-2
            list: geomesa-accumulo/geomesa-accumulo-datastore -Dtest="AccumuloDataStoreAttributeVisibilityTest,AttributeIndexFilteringIteratorTest,AccumuloPartitioningTest,AccumuloFeatureReaderTest,AccumuloDataStoreAtomicWriteTest,AttributeIndexValuesTest,AccumuloDataStoreColumnGroupsTest,ZIntervalTest,S2IndexTest,AccumuloFeatureWriterTest,KryoLazyStatsIteratorTest,AccumuloDataStoreSortTest,AccumuloDataStoreUuidTest,AccumuloDataStoreAuthTest,DensityIteratorTest,S3IndexTest"
          - name: accumulo-ds-3
            list: geomesa-accumulo/geomesa-accumulo-datastore -Dtest.excludes="AccumuloDataStoreTest,AccumuloDataStoreQueryTest,AccumuloDataStoreStatsTest,AttributeIndexStrategyTest,AccumuloDataStoreAlterSchemaTest,ArrowBatchIteratorTest,MultiIteratorTest,AccumuloDataStoreTransformsTest,ArrowDictionaryTest,VisibilitiesTest,AccumuloDataStoreAttributeVisibilityTest,AttributeIndexFilteringIteratorTest,AccumuloPartitioningTest,AccumuloFeatureReaderTest,AccumuloDataStoreAtomicWriteTest,AttributeIndexValuesTest,AccumuloDataStoreColumnGroupsTest,ZIntervalTest,S2IndexTest,AccumuloFeatureWriterTest,KryoLazyStatsIteratorTest,AccumuloDataStoreSortTest,AccumuloDataStoreUuidTest,AccumuloDataStoreAuthTest,DensityIteratorTest,S3IndexTest"
          - name: accumulo-other
            # we exclude distributed runtime as it breaks the classpath loading, but it doesn't have any tests anyway
            list: geomesa-accumulo/geomesa-accumulo-dist,geomesa-accumulo/geomesa-accumulo-gs-plugin,geomesa-accumulo/geomesa-accumulo-indices,geomesa-accumulo/geomesa-accumulo-iterators,geomesa-accumulo/geomesa-accumulo-jobs,geomesa-accumulo/geomesa-accumulo-spark,geomesa-accumulo/geomesa-accumulo-tools
          - name: arrow
            list: geomesa-arrow/geomesa-arrow-datastore,geomesa-arrow/geomesa-arrow-dist,geomesa-arrow/geomesa-arrow-gt,geomesa-arrow/geomesa-arrow-jts,geomesa-arrow/geomesa-arrow-tools
          - name: cassandra
            list: geomesa-cassandra/geomesa-cassandra-datastore,geomesa-cassandra/geomesa-cassandra-dist,geomesa-cassandra/geomesa-cassandra-gs-plugin,geomesa-cassandra/geomesa-cassandra-tools
          - name: convert
            list: geomesa-convert/geomesa-convert-all,geomesa-convert/geomesa-convert-avro,geomesa-convert/geomesa-convert-avro-schema-registry,geomesa-convert/geomesa-convert-common,geomesa-convert/geomesa-convert-fixedwidth,geomesa-convert/geomesa-convert-jdbc,geomesa-convert/geomesa-convert-json,geomesa-convert/geomesa-convert-parquet,geomesa-convert/geomesa-convert-redis-cache,geomesa-convert/geomesa-convert-shp,geomesa-convert/geomesa-convert-simplefeature,geomesa-convert/geomesa-convert-text,geomesa-convert/geomesa-convert-xml
          - name: features
            list: geomesa-features/geomesa-feature-all,geomesa-features/geomesa-feature-avro,geomesa-features/geomesa-feature-common,geomesa-features/geomesa-feature-exporters,geomesa-features/geomesa-feature-kryo
          - name: fs
            list: geomesa-fs/geomesa-fs-datastore,geomesa-fs/geomesa-fs-dist,geomesa-fs/geomesa-fs-gs-plugin,geomesa-fs/geomesa-fs-spark,geomesa-fs/geomesa-fs-tools
          - name: fs-storage
            list: geomesa-fs/geomesa-fs-storage/geomesa-fs-storage-api,geomesa-fs/geomesa-fs-storage/geomesa-fs-storage-common,geomesa-fs/geomesa-fs-storage/geomesa-fs-storage-convert,geomesa-fs/geomesa-fs-storage/geomesa-fs-storage-parquet,geomesa-fs/geomesa-fs-storage/geomesa-fs-storage-parquet-io
          - name: gt-pg15
            list: geomesa-gt/geomesa-gt-dist,geomesa-gt/geomesa-gt-gs-plugin,geomesa-gt/geomesa-gt-partitioning,geomesa-gt/geomesa-gt-spark,geomesa-gt/geomesa-gt-tools -Dtest.postgis.docker.tag=15-3.4
          - name: gt-pg16
            list: geomesa-gt/geomesa-gt-dist,geomesa-gt/geomesa-gt-gs-plugin,geomesa-gt/geomesa-gt-partitioning,geomesa-gt/geomesa-gt-spark,geomesa-gt/geomesa-gt-tools -Dtest.postgis.docker.tag=16-3.5
          - name: gt-pg17
            list: geomesa-gt/geomesa-gt-dist,geomesa-gt/geomesa-gt-gs-plugin,geomesa-gt/geomesa-gt-partitioning,geomesa-gt/geomesa-gt-spark,geomesa-gt/geomesa-gt-tools -Dtest.postgis.docker.tag=17-3.5
          - name: hbase-ds
            list: geomesa-hbase/geomesa-hbase-datastore
          - name: hbase-other
            # we exclude distributed runtime as it breaks the classpath loading, but it doesn't have any tests anyway
            list: geomesa-hbase/geomesa-hbase-dist,geomesa-hbase/geomesa-hbase-gs-plugin,geomesa-hbase/geomesa-hbase-jobs,geomesa-hbase/geomesa-hbase-rpc,geomesa-hbase/geomesa-hbase-server,geomesa-hbase/geomesa-hbase-server-hbase2,geomesa-hbase/geomesa-hbase-spark,geomesa-hbase/geomesa-hbase-tools
          - name: kafka
            list: geomesa-kafka/geomesa-kafka-confluent,geomesa-kafka/geomesa-kafka-datastore,geomesa-kafka/geomesa-kafka-dist,geomesa-kafka/geomesa-kafka-gs-plugin,geomesa-kafka/geomesa-kafka-tools,geomesa-kafka/geomesa-kafka-utils
          - name: lambda
            list: geomesa-lambda/geomesa-lambda-datastore,geomesa-lambda/geomesa-lambda-dist,geomesa-lambda/geomesa-lambda-gs-plugin,geomesa-lambda/geomesa-lambda-tools
          - name: misc-1
            list: geomesa-memory/geomesa-cqengine,geomesa-memory/geomesa-cqengine-datastore,geomesa-metrics/geomesa-metrics-micrometer,geomesa-process/geomesa-process-vector
          - name: misc-2
            list: geomesa-filter,geomesa-index-api,geomesa-jobs,geomesa-security,geomesa-tools,geomesa-z3
          - name: redis
            list: geomesa-redis/geomesa-redis-datastore,geomesa-redis/geomesa-redis-dist,geomesa-redis/geomesa-redis-gs-plugin,geomesa-redis/geomesa-redis-tools
          - name: spark
            list: geomesa-spark/geomesa_pyspark,geomesa-spark/geomesa-spark-converter,geomesa-spark/geomesa-spark-core,geomesa-spark/geomesa-spark-jts,geomesa-spark/geomesa-spark-jupyter-leaflet,geomesa-spark/geomesa-spark-sql,geomesa-spark/geomesa-spark-test-with-sedona
          - name: utils
            list: geomesa-utils-parent/geomesa-bom,geomesa-utils-parent/geomesa-hadoop-utils,geomesa-utils-parent/geomesa-utils
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
        with:
          distribution: temurin
          java-version: "${{ env.JAVA_VERSION }}"
      - uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
        with:
          key: ${{ hashFiles('**/pom.xml') }}-build-${{ matrix.scala-version }}-${{ matrix.projects.name }}
          path: ~/.m2/repository/
      - name: Set Scala version
        run: ./build/scripts/change-scala-version.sh ${{ matrix.scala-version }}
      - name: Compile
        id: compile
        continue-on-error: true
        run: |
          set -o pipefail
          mvn $MAVEN_COMPILE_ARGS $MAVEN_CLI_OPTS -am -pl ${{ matrix.projects.list }} | tee -a build.log
      - name: Compile (retry)
        if: steps.compile.outcome=='failure'
        run: |
          set -o pipefail
          # retry if the failure was due to transient download errors from maven central
          if grep -q -e 'Could not transfer artifact' -e 'Failed to read artifact descriptor' build.log; then
            RESUME_FROM="$({ grep --text 'mvn <args> -rf ' build.log || test $? = 1; } | tail -n1 | sed 's/.*-rf/-rf/')"
            mvn $MAVEN_COMPILE_ARGS $MAVEN_CLI_OPTS -am -pl ${{ matrix.projects.list }} $RESUME_FROM | tee -a build.log
          else
            exit 1
          fi
      - name: Run unit tests
        id: test
        continue-on-error: true
        run: |
          set -o pipefail
          mvn $MAVEN_TEST_ARGS $MAVEN_CLI_OPTS -pl ${{ matrix.projects.list }} | tee -a test.log
      - name: Run unit tests (retry)
        id: test-retry
        if: steps.test.outcome=='failure'
        run: |
          set -o pipefail
          RESUME_FROM="$({ grep --text 'mvn <args> -rf ' test.log || test $? = 1; } | tail -n1 | sed 's/.*-rf/-rf/')"
          mvn $MAVEN_TEST_ARGS $MAVEN_CLI_OPTS -pl ${{ matrix.projects.list }} $RESUME_FROM | tee -a test.log
      - name: Remove geomesa artifacts
        if: success() || failure()
        run: rm -rf ~/.m2/repository/org/locationtech/geomesa