Run summaries #1467
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Daily: refresh NJSP data, update www" | |
| on: | |
| schedule: | |
| - cron: "10 15 * * *" | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| inputs: | |
| force: | |
| type: boolean | |
| description: "Force re-run all stages (ignore freshness)" | |
| default: false | |
| targets: | |
| type: string | |
| description: "Specific DVX targets to run (space-separated .dvc paths, empty = all)" | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: us-east-1 | |
| GH_TOKEN: ${{ secrets.GH_TOKEN }} | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} | |
| jobs: | |
| run: | |
| name: DVX pipeline | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ github.ref_name }} | |
| token: ${{ secrets.GH_TOKEN }} | |
| - uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| - name: Install Python dependencies | |
| run: | | |
| uv sync | |
| echo "$PWD/.venv/bin" >> $GITHUB_PATH | |
| - uses: pnpm/action-setup@v5 | |
| with: | |
| version: 10 | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version: 22 | |
| cache: 'pnpm' | |
| cache-dependency-path: www/pnpm-lock.yaml | |
| - name: Install www dependencies | |
| run: cd www && pnpm install | |
| - name: Configure Git author | |
| run: | | |
| git config --global user.name 'GitHub Actions' | |
| git config --global user.email 'ryan-williams@users.noreply.github.com' | |
| - name: DVX pull (fetch DVC-tracked deps from S3) | |
| run: dvx pull njsp/data/crashes.parquet | |
| - name: DVX status | |
| run: dvx status | |
| - name: DVX run | |
| run: | | |
| args="" | |
| if [ "${{ inputs.force }}" = "true" ]; then args="$args --force"; fi | |
| if [ -n "${{ inputs.targets }}" ]; then | |
| args="$args ${{ inputs.targets }}" | |
| else | |
| # Default targets: daily NJSP pipeline (not full NJDOT raw data rebuild) | |
| args="$args njsp/data/refresh.dvc njsp/data/harmonize.dvc njsp/data/summaries.dvc www/public/njsp/csvs.dvc www/public/njsp/projections.dvc www/deploy.dvc api/d1-import.dvc" | |
| fi | |
| dvx run --commit --push each $args |