Issue#38 Add targeted MySQL↔BigQuery CI coverage #142
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI-COVER-VERSIONS | |
| on: | |
| pull_request: | |
| workflow_dispatch: | |
| jobs: | |
| unit_tests: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| python-version: | |
| - "3.9" | |
| - "3.10" | |
| - "3.11" | |
| - "3.12" | |
| name: Check Python ${{ matrix.python-version }} on ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Packages | |
| run: chmod +x tests/install-packages.sh && ./tests/install-packages.sh | |
| - name: Ensure setuptools is installed | |
| run: pip install --upgrade pip setuptools | |
| - name: Build the stack | |
| run: docker-compose up -d mysql postgres presto trino clickhouse | |
| - name: Install uv | |
| run: pip install uv | |
| - name: Install package | |
| run: | | |
| pip install setuptools | |
| uv sync --all-extras --group ci --verbose | |
| - name: Debug Environment | |
| run: | | |
| echo "--- ls -la ---" | |
| ls -la | |
| echo "--- pip freeze ---" | |
| pip freeze | |
| echo "--- PYTHONPATH ---" | |
| echo $PYTHONPATH | |
| echo "--- which python ---" | |
| which python | |
| echo "--- pkg_resources location ---" | |
| python -c "import pkg_resources; print(pkg_resources.__file__)" | |
| # BigQuery start | |
| # - id: 'auth' | |
| # uses: 'google-github-actions/auth@v1' | |
| # with: | |
| # credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}' | |
| # - name: 'Set up BigQuery Cloud SDK' | |
| # uses: 'google-github-actions/setup-gcloud@v1' | |
| # - name: 'Use gcloud CLI' | |
| # run: "gcloud config configurations list" | |
| - name: 'Create datadiff SA' | |
| run: "echo '${{ secrets.BQ_SA }}' > bq_sa.json" | |
| # BigQuery end | |
| - name: Run unit tests | |
| env: | |
| DATADIFF_SNOWFLAKE_URI: '${{ secrets.DATADIFF_SNOWFLAKE_URI }}' | |
| DATADIFF_PRESTO_URI: '${{ secrets.DATADIFF_PRESTO_URI }}' | |
| DATADIFF_TRINO_URI: '${{ secrets.DATADIFF_TRINO_URI }}' | |
| GOOGLE_APPLICATION_CREDENTIALS: 'bq_sa.json' | |
| DATADIFF_BIGQUERY_URI: '${{ secrets.DATADIFF_BIGQUERY_URI }}' | |
| DATADIFF_CLICKHOUSE_URI: 'clickhouse://clickhouse:Password1@localhost:9000/clickhouse' | |
| DATADIFF_REDSHIFT_URI: '${{ secrets.DATADIFF_REDSHIFT_URI }}' | |
| MOTHERDUCK_TOKEN: '${{ secrets.MOTHERDUCK_TOKEN }}' | |
| run: | | |
| chmod +x tests/waiting_for_stack_up.sh | |
| ./tests/waiting_for_stack_up.sh | |
| source .venv/bin/activate | |
| TEST_ACROSS_ALL_DBS=0 unittest-parallel -j 16 |