Additional cache #40
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI for testing-ci branch | |
| on: | |
| push: | |
| branches: | |
| - testing-ci | |
| workflow_dispatch: | |
| env: | |
| # Centralize versions for easier maintenance | |
| PYTHON_VERSION: "3.10" | |
| WEAVIATE_VERSION: "1.32.0" | |
| OLLAMA_VERSION: "0.9.6" | |
| CLIP_MODEL_TAG: "sentence-transformers-clip-ViT-B-32" | |
| jobs: | |
| test: | |
| runs-on: ubuntu-latest | |
| # Set timeout for the entire job | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| # Consolidated caching step for pip and venv | |
| - name: Cache Python dependencies | |
| uses: actions/cache@v4 | |
| id: cache-python | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| .venv | |
| key: ${{ runner.os }}-python-${{ env.PYTHON_VERSION }}-${{ hashFiles('**/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-python-${{ env.PYTHON_VERSION }}- | |
| - name: Install Python dependencies | |
| run: | | |
| echo "📦 Setting up Python environment..." | |
| if [[ "${{ steps.cache-python.outputs.cache-hit }}" != "true" ]]; then | |
| echo "Creating virtual environment and installing dependencies..." | |
| python -m venv .venv | |
| fi | |
| source .venv/bin/activate | |
| pip install --upgrade pip | |
| pip install -r tests/requirements.txt | |
| echo "✅ Python environment ready. Installed packages:" | |
| pip list | head -20 | |
| # Parallel Docker image caching - define cache paths | |
| - name: Cache Docker images | |
| uses: actions/cache@v4 | |
| id: cache-docker-images | |
| with: | |
| path: | | |
| /tmp/weaviate-base.tar | |
| /tmp/clip-image.tar | |
| /tmp/ollama-base.tar | |
| key: ${{ runner.os }}-docker-images-${{ env.WEAVIATE_VERSION }}-${{ env.CLIP_MODEL_TAG }}-${{ env.OLLAMA_VERSION }} | |
| restore-keys: | | |
| ${{ runner.os }}-docker-images- | |
| - name: Load or pull Docker images | |
| run: | | |
| echo "🐳 Managing Docker images..." | |
| echo "Initial disk space:" | |
| df -h | grep -E "^/dev|Filesystem" | |
| # Function to handle image loading/pulling | |
| load_or_pull_image() { | |
| local image_name=$1 | |
| local tar_file=$2 | |
| local cache_hit=$3 | |
| if [[ "$cache_hit" == "true" ]] && [[ -f "$tar_file" ]]; then | |
| echo "✅ Loading $image_name from cache..." | |
| docker load --input "$tar_file" | |
| else | |
| echo "⬇️ Pulling $image_name..." | |
| docker pull "$image_name" | |
| docker save "$image_name" -o "$tar_file" | |
| fi | |
| } | |
| # Process images | |
| load_or_pull_image "cr.weaviate.io/semitechnologies/weaviate:${{ env.WEAVIATE_VERSION }}" \ | |
| "/tmp/weaviate-base.tar" \ | |
| "${{ steps.cache-docker-images.outputs.cache-hit }}" | |
| load_or_pull_image "cr.weaviate.io/semitechnologies/multi2vec-clip:${{ env.CLIP_MODEL_TAG }}" \ | |
| "/tmp/clip-image.tar" \ | |
| "${{ steps.cache-docker-images.outputs.cache-hit }}" | |
| load_or_pull_image "ollama/ollama:${{ env.OLLAMA_VERSION }}" \ | |
| "/tmp/ollama-base.tar" \ | |
| "${{ steps.cache-docker-images.outputs.cache-hit }}" | |
| echo "✅ All Docker images ready" | |
| echo "Final disk space:" | |
| df -h | grep -E "^/dev|Filesystem" | |
| - name: Cache Ollama models | |
| uses: actions/cache@v4 | |
| id: cache-ollama-models | |
| with: | |
| path: ~/.ollama | |
| key: ${{ runner.os }}-ollama-models-snowflake-nomic-llama32-v2 | |
| restore-keys: | | |
| ${{ runner.os }}-ollama-models- | |
| - name: Start services | |
| run: | | |
| echo "🚀 Starting Weaviate and Ollama services..." | |
| mkdir -p ~/.ollama | |
| ./tests/start-weaviate.sh | |
| # Wait for Ollama to be ready before pulling models | |
| sleep 5 # Wait for Ollama service to be ready | |
| echo "✅ Services started:" | |
| docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" | |
| - name: Ensure Ollama models are available | |
| run: | | |
| echo "🤖 Checking Ollama models..." | |
| declare -a models=("snowflake-arctic-embed" "nomic-embed-text" "llama3.2") | |
| for model in "${models[@]}"; do | |
| if docker exec tests-ollama-1 ollama list 2>/dev/null | grep -q "$model"; then | |
| echo "✅ Model $model already available" | |
| else | |
| echo "⬇️ Pulling model: $model" | |
| docker exec tests-ollama-1 ollama pull "$model" | |
| fi | |
| done | |
| echo "✅ All models ready:" | |
| docker exec tests-ollama-1 ollama list | |
| - name: Wait for all services to be ready | |
| run: | | |
| echo "⏳ Waiting for services to be ready..." | |
| # Define services with their health check endpoints | |
| declare -A services=( | |
| ["8099"]="Weaviate Main" | |
| ["8580"]="Weaviate GRPC" | |
| ["8080"]="Weaviate Instance 1" | |
| ["8090"]="Weaviate Instance 2" | |
| ["8280"]="Weaviate Instance 3" | |
| ["8180"]="Weaviate Instance 4" | |
| ["8181"]="Weaviate Instance 5" | |
| ["8182"]="Weaviate Instance 6" | |
| ) | |
| for port in "${!services[@]}"; do | |
| service_name="${services[$port]}" | |
| echo -n "Checking $service_name (port $port)... " | |
| if timeout 60 bash -c "until curl -sf http://localhost:$port/v1/.well-known/ready &>/dev/null; do sleep 2; done"; then | |
| echo "✅ Ready" | |
| else | |
| echo "⚠️ Not responding (may be optional)" | |
| fi | |
| done | |
| echo "✅ Service check completed" | |
| - name: Run tests | |
| env: | |
| DOCS_GITHUB_ENV: "true" | |
| # Weaviate connection settings | |
| WEAVIATE_URL: ${{ secrets.WEAVIATE_URL }} | |
| WEAVIATE_HOST: ${{ secrets.WEAVIATE_HOST }} | |
| WEAVIATE_HOSTNAME: ${{ secrets.WEAVIATE_HOSTNAME }} | |
| WEAVIATE_HTTP_HOST: ${{ secrets.WEAVIATE_HTTP_HOST }} | |
| WEAVIATE_GRPC_HOST: ${{ secrets.WEAVIATE_GRPC_HOST }} | |
| WEAVIATE_API_KEY: ${{ secrets.WEAVIATE_API_KEY }} | |
| WEAVIATE_LOCAL_API_KEY: ${{ secrets.WEAVIATE_LOCAL_API_KEY }} | |
| # API Keys - using consistent naming | |
| COHERE_APIKEY: ${{ secrets.COHERE_APIKEY }} | |
| OPENAI_APIKEY: ${{ secrets.OPENAI_APIKEY }} | |
| HUGGINGFACE_APIKEY: ${{ secrets.HUGGINGFACE_APIKEY }} | |
| ANTHROPIC_APIKEY: ${{ secrets.ANTHROPIC_APIKEY }} | |
| # WCD credentials | |
| WCD_USERNAME: ${{ secrets.WCD_USERNAME }} | |
| WCD_PASSWORD: ${{ secrets.WCD_PASSWORD }} | |
| run: | | |
| echo "🧪 Running tests..." | |
| source .venv/bin/activate | |
| pytest -m pyv4 -v --tb=short --durations=10 | |
| echo "✅ Tests completed" | |
| # Cleanup steps with consistent error handling | |
| - name: Stop services | |
| if: always() | |
| run: | | |
| echo "🛑 Stopping services..." | |
| ./tests/stop-weaviate.sh 2>/dev/null || echo "⚠️ Some services may have already stopped" | |
| - name: Cleanup Docker resources | |
| if: always() | |
| run: | | |
| echo "🧹 Cleaning up Docker resources..." | |
| # Stop any remaining containers from compose files | |
| for compose_file in tests/docker-compose*.yml; do | |
| if [[ -f "$compose_file" ]]; then | |
| docker compose -f "$compose_file" down -v --remove-orphans 2>/dev/null || true | |
| fi | |
| done | |
| # Clean up dangling resources | |
| docker system prune -f --volumes 2>/dev/null || true | |
| echo "✅ Cleanup completed" | |
| - name: Upload test results | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-results | |
| path: | | |
| pytest-results.xml | |
| .pytest_cache/ | |
| retention-days: 7 |