chore: CI/testing improvements for parallel test execution (#317)

* chore: CI/testing improvements for parallel test execution

- Add workflows-dev pytest command with parallel test runner
- Add pytest-xdist for parallel test execution (-nauto)
- Add timeouts to prevent test hangs (--timeout=10/60/120)
- Use module-scoped async fixtures for faster tests
- Remove mypy in favor of basedpyright/ty type checking
- Add test-docker CI job for integration tests with testcontainers
- Exclude integration tests from Python 3.9/3.14 in CI matrix

Author: adrianlyjak

.github/workflows/publish_openapi.yml

@@ -39,7 +39,7 @@ jobs:
       - name: Compute release metadata
         id: metadata
         run: >
-          uv run --package workflows-dev workflows-dev compute-tag-metadata
+          uv run dev compute-tag-metadata
           --tag "${{ env.TARGET_TAG }}"
           --output "$GITHUB_OUTPUT"
 

.github/workflows/test.yml

@@ -21,29 +21,35 @@ jobs:
           [
             llama-index-workflows,
             llama-index-utils-workflow,
-            workflows-dev,
             llama-index-integration-tests,
+            llama-agents-dev,
           ]
         exclude:
-          - package: workflows-dev
+          # Integration tests on 3.14 run in test-docker job with cross-package coverage
+          - package: llama-index-integration-tests
+            python-version: "3.9"
+          - package: llama-index-integration-tests
+            python-version: "3.14"
+          # Dev CLI only needs to run on 3.14
+          - package: llama-agents-dev
             python-version: "3.9"
-          - package: workflows-dev
+          - package: llama-agents-dev
             python-version: "3.10"
-          - package: workflows-dev
+          - package: llama-agents-dev
             python-version: "3.11"
-          - package: workflows-dev
+          - package: llama-agents-dev
             python-version: "3.12"
-          - package: workflows-dev
+          - package: llama-agents-dev
             python-version: "3.13"
         include:
           - package: llama-index-workflows
             package_dir: packages/llama-index-workflows
           - package: llama-index-utils-workflow
             package_dir: packages/llama-index-utils-workflow
-          - package: workflows-dev
-            package_dir: packages/workflows-dev
           - package: llama-index-integration-tests
             package_dir: packages/llama-index-integration-tests
+          - package: llama-agents-dev
+            package_dir: .
 
     steps:
       - uses: actions/checkout@v4
@@ -58,7 +64,9 @@ jobs:
 
       - name: Run tests
         if: matrix.python-version != env.COV_PYTHON_VERSION
-        run: uv run --all-extras --directory ${{ matrix.package_dir }} -- pytest
+        run: |
+          uv sync --python ${{ matrix.python-version }} --all-extras --directory ${{ matrix.package_dir }}
+          uv run --python ${{ matrix.python-version }} --all-extras --directory ${{ matrix.package_dir }} -- pytest
 
       - name: Run tests with coverage
         if: matrix.python-version == env.COV_PYTHON_VERSION
@@ -74,8 +82,37 @@ jobs:
         env:
           COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
 
+  test-docker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install uv and set the python version
+        uses: astral-sh/setup-uv@v6
+        with:
+          python-version: "3.14"
+          enable-cache: true
+
+      - name: Run non-Docker tests with coverage
+        run: uv run --all-extras --directory packages/llama-index-integration-tests -- pytest -m "not docker" --cov --cov-report=
+
+      - name: Run Docker tests with coverage (append)
+        # Use longer timeout for Docker tests since container startup can take 30-60s in CI
+        run: uv run --all-extras --directory packages/llama-index-integration-tests -- pytest --timeout=120 -m docker --cov --cov-append --cov-report=xml
+
+      - name: Report Coveralls
+        uses: coverallsapp/github-action@v2
+        with:
+          file: packages/llama-index-integration-tests/coverage.xml
+          flag-name: llama-index-integration-tests
+          parallel: true
+        env:
+          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
+
   coveralls-finish:
-    needs: test
+    needs: [test, test-docker]
     runs-on: ubuntu-latest
     if: github.repository == 'run-llama/workflows-py'
     steps:

.github/workflows/update_debugger_assets.yml

@@ -42,7 +42,7 @@ jobs:
 
       - name: Update index.html
         run: >
-          uv run --package workflows-dev workflows-dev update-index-html
+          uv run dev update-index-html
           --js-url "${{ steps.payload.outputs.js_url }}"
           --css-url "${{ steps.payload.outputs.css_url }}"
 

.pre-commit-config.yaml

@@ -28,32 +28,8 @@ repos:
       - id: ty
         name: ty
         language: system
-        entry: uv run ty check packages
+        entry: uv run ty check packages/
         pass_filenames: false
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.15.0
-    hooks:
-      - id: mypy
-        exclude: ^packages/(llama-index-workflows-server|llama-index-workflows-client|llama-index-utils-workflow|workflows-dev|llama-index-integration-tests)/tests/|^packages/llama-index-workflows/src/llama_agents/|^packages/llama-index-workflows/tests/test_llama_agents_alias\.py
-        additional_dependencies:
-          [
-            "types-Deprecated",
-            "types-PyYAML",
-            "types-botocore",
-            "types-aiobotocore",
-            "types-protobuf==4.24.0.4",
-            "types-redis",
-            "types-requests",
-            "types-setuptools",
-            "types-click",
-          ]
-        args:
-          [
-            --disallow-untyped-defs,
-            --ignore-missing-imports,
-            --python-version=3.11,
-          ]
-
   - repo: https://github.com/DetachHead/basedpyright-pre-commit-mirror
     rev: 1.31.1
     hooks:

AGENTS.md

@@ -13,24 +13,24 @@ This is the LlamaIndex Workflows library - an event-driven, async-first framewor
 ## Development Commands
 
 ### Testing
-```bash
-# Run all tests
-uv run --directory packages/llama-index-workflows pytest
 
-# Run tests with coverage
-uv run --directory packages/llama-index-workflows pytest --cov=src/workflows --cov-report=html
+Use the `dev` CLI to run tests:
+
+```bash
+# Run all package tests
+uv run dev
 
-# Run specific test files
-uv run --directory packages/llama-index-workflows pytest tests/test_server.py tests/test_server_utils.py
+# Filter by substring match
+uv run dev -p workflows
+uv run dev -p server -p client
 
-# Run tests in verbose mode
-uv run --directory packages/llama-index-workflows pytest -v
+# Pass pytest args after --
+uv run dev -- -k test_name
 ```
 
 ### Linting & Formatting
 ```bash
-# Run pre-commit hooks
-uv run --directory packages/llama-index-workflows pre-commit run -a
+uv run pre-commit run -a
 ```
 
 ## Project Structure
@@ -46,7 +46,7 @@ uv run --directory packages/llama-index-workflows pre-commit run -a
 - **WorkflowServer** - HTTP server for serving workflows as web services
 
 ## Notes for Claude
-- Always run tests after making changes: `uv run --directory packages/llama-index-workflows pytest`
+- Always run tests after making changes: `uv run dev`
 - Never use classes for tests, only use pytest functions
 - Always annotate with types function arguments and return values
 - The project uses async/await extensively
@@ -62,17 +62,11 @@ Make sure to install uv as the package manager. Development commands rely on it.
 curl -fsSL https://astral.sh/uv/install.sh | sh
 ```
 
-Always run test tests and pre-commit commands before committing. They run very fast and are not verbose.
-
-Tests:
+Always run tests and pre-commit before committing:
 
 ```bash
-uv run --directory packages/llama-index-workflows pytest -nauto --timeout=1
-```
-
-Linting, typechecking, and formatting:
-```bash
-uv run --directory packages/llama-index-workflows pre-commit run -a
+uv run dev
+uv run pre-commit run -a
 ```
 
 ## Testing Patterns
@@ -84,3 +78,10 @@ We use **pytest** with idiomatic pytest patterns. Follow these guidelines:
 - **Prefer Real Objects Over Mocks**: Use simple dataclasses and real objects directly when available rather than mocking them. Only mock external dependencies or things that are truly difficult to instantiate.
 - **DRY Test Setup**: Do not repeat patches or setup code. Create reusable abstractions—fixtures, helper functions, or module-level constants—that can be shared across tests. Tests can easily be overwhelmed with setup; start from a rich suite of testing utilities to enable many small, expressive tests.
 - **Simple Testing Utilities**: Testing utilities should be basic—just functions, fixtures, and global variables. Avoid over-engineering test infrastructure.
+
+## Coding Style
+
+- Always use `from __future__ import annotations` at the top of each test file. Never use string annotations.
+- Include the standard SPDX license header at the top of each test file.
+- Comments are useful, but avoid fluff.
+- Never use inline imports unless required to prevent circular dependencies.

CONTRIBUTING.md

@@ -47,10 +47,18 @@ uv run pre-commit install
 
 ## Run tests
 
-Tests are run with `pytest`. You can run them with:
+Use the `dev` CLI to run tests across packages:
 
 ```bash
-uv run pytest
+# Run all package tests
+uv run dev
+
+# Filter by substring match
+uv run dev -p workflows
+uv run dev -p server -p client
+
+# Pass pytest args after --
+uv run dev -- -k test_name
 ```
 
 Generally, all features should be covered by robust tests. If you are adding a new feature or fixing a bug, please add tests for it.

packages/llama-index-integration-tests/pyproject.toml

@@ -4,10 +4,16 @@ build-backend = "uv_build"
 
 [dependency-groups]
 dev = [
+  "basedpyright>=1.31.1",
+  "psycopg[binary]>=3.2.0",
   "pytest>=8.4.2",
   "pytest-asyncio>=0.24.0",
   "pytest-cov>=7.0.0",
-  "pytest-timeout>=2.3.1"
+  "pytest-timeout>=2.3.1",
+  "pytest-xdist>=3.8.0",
+  "sqlalchemy>=2.0.0",
+  "testcontainers[postgres]>=4.0.0",
+  "ty>=0.0.1,<0.0.9"
 ]
 
 [project]
@@ -21,10 +27,31 @@ dependencies = [
   "llama-index-workflows"
 ]
 
+[tool.basedpyright]
+typeCheckingMode = "standard"
+pythonVersion = "3.10"
+
+[tool.coverage.run]
+source = [
+  "workflows",
+  "llama_agents"
+]
+omit = ["**/tests/*"]
+
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
-asyncio_default_fixture_loop_scope = "function"
+asyncio_default_fixture_loop_scope = "module"
+asyncio_default_test_loop_scope = "module"
 testpaths = ["tests"]
+# Skip docker tests by default - run with `pytest -m docker` to include them
+addopts = "-nauto --timeout=60 -m 'not docker'"
+markers = [
+  "docker: marks tests as requiring Docker (testcontainers/PostgreSQL)"
+]
+filterwarnings = [
+  # Ignore internal testcontainers deprecation warning (fixed in their code, just noisy)
+  "ignore:The @wait_container_is_ready decorator is deprecated:DeprecationWarning"
+]
 
 [tool.uv.sources]
 llama-index-workflows = {workspace = true}

packages/llama-index-integration-tests/tests/conftest.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from typing import Any, Callable, List, Protocol, Union
+from typing import Any, Callable, Generator, List, Protocol, Union
 
 import pytest
 from llama_index.core.agent.workflow import (
@@ -17,6 +17,8 @@
     make_text_response,
     response_generator_from_list,
 )
+from sqlalchemy.engine import Engine
+from testcontainers.postgres import PostgresContainer
 
 
 class WorkflowFactory(Protocol):
@@ -132,3 +134,39 @@ def _create(
         )
 
     return _create  # type: ignore[return-value]
+
+
+# -- Docker/PostgreSQL Fixtures --
+
+
+@pytest.fixture(scope="module")
+def postgres_container() -> Generator[PostgresContainer, None, None]:
+    """Module-scoped PostgreSQL container for integration tests.
+
+    Requires Docker to be running. Used by tests marked with @pytest.mark.docker.
+    """
+    with PostgresContainer("postgres:16", driver=None) as postgres:
+        yield postgres
+
+
+@pytest.fixture(scope="module")
+def postgres_engine(
+    postgres_container: PostgresContainer,
+) -> Generator[Engine, None, None]:
+    """Module-scoped PostgreSQL engine for integration tests."""
+    from sqlalchemy import create_engine
+
+    # Get connection URL and convert to use psycopg (psycopg3) driver
+    connection_url = postgres_container.get_connection_url()
+    # Replace postgresql:// or postgresql+psycopg2:// with postgresql+psycopg://
+    if "postgresql+psycopg2://" in connection_url:
+        connection_url = connection_url.replace(
+            "postgresql+psycopg2://", "postgresql+psycopg://"
+        )
+    elif connection_url.startswith("postgresql://"):
+        connection_url = connection_url.replace(
+            "postgresql://", "postgresql+psycopg://", 1
+        )
+    engine = create_engine(connection_url)
+    yield engine
+    engine.dispose()

packages/llama-index-integration-tests/tests/test_docker_postgres.py

@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 LlamaIndex Inc.
+
+"""Basic Docker/PostgreSQL connectivity test."""
+
+from __future__ import annotations
+
+import pytest
+from sqlalchemy import text
+from sqlalchemy.engine import Engine
+
+
+@pytest.mark.docker
+def test_postgres_connection(postgres_engine: Engine) -> None:
+    """Test basic PostgreSQL connectivity with SELECT 1."""
+    with postgres_engine.connect() as conn:
+        result = conn.execute(text("SELECT 1"))
+        assert result.scalar() == 1

packages/llama-index-utils-workflow/pyproject.toml

@@ -7,7 +7,9 @@ dev = [
   "pre-commit>=4.3.0",
   "pytest>=8.4.2",
   "pytest-asyncio>=1.0.0",
-  "pytest-cov>=6.1.1"
+  "pytest-cov>=6.1.1",
+  "pytest-timeout>=2.4.0",
+  "pytest-xdist>=3.8.0"
 ]
 
 [project]
@@ -23,5 +25,12 @@ dependencies = [
   "pyvis>=0.3.2"
 ]
 
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "module"
+asyncio_default_test_loop_scope = "module"
+testpaths = ["tests"]
+addopts = "-nauto --timeout=10"
+
 [tool.uv.build-backend]
 module-name = "llama_index.utils.workflow"