Additional cache #39
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI for testing-ci branch | |
| on: | |
| push: | |
| branches: | |
| - testing-ci | |
| workflow_dispatch: | |
| jobs: | |
| test: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| # Free up disk space by removing unnecessary software | |
| - name: Free disk space | |
| run: | | |
| echo "=== Initial disk usage ===" | |
| df -h | |
| echo "=== Removing unnecessary software ===" | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /usr/local/share/boost | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| sudo rm -rf /opt/hostedtoolcache | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /usr/local/share/powershell | |
| sudo rm -rf /usr/local/share/chromium | |
| sudo rm -rf /usr/local/lib/node_modules | |
| echo "=== Docker cleanup ===" | |
| docker system prune -af --volumes | |
| docker builder prune -af | |
| echo "=== APT cleanup ===" | |
| sudo apt-get autoremove -y | |
| sudo apt-get autoclean | |
| sudo rm -rf /var/lib/apt/lists/* | |
| echo "=== Final disk usage ===" | |
| df -h | |
| # Make scripts executable | |
| - name: Make scripts executable | |
| run: | | |
| chmod +x tests/start-weaviate.sh | |
| chmod +x tests/stop-weaviate.sh | |
| # Set up Python BEFORE Docker operations to reduce memory pressure | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| # Cache pip dependencies | |
| - name: Cache pip dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| # Cache virtual environment | |
| - name: Cache virtual environment | |
| uses: actions/cache@v4 | |
| id: cache-venv | |
| with: | |
| path: .venv | |
| key: ${{ runner.os }}-venv-${{ runner.arch }}-python3.10-${{ hashFiles('**/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-venv-${{ runner.arch }}-python3.10- | |
| - name: Create virtual environment and install dependencies | |
| if: steps.cache-venv.outputs.cache-hit != 'true' | |
| run: | | |
| python -m venv .venv | |
| source .venv/bin/activate | |
| pip install --upgrade pip | |
| pip install -r tests/requirements.txt | |
| - name: Verify cached virtual environment | |
| if: steps.cache-venv.outputs.cache-hit == 'true' | |
| run: | | |
| source .venv/bin/activate | |
| python --version | |
| pip list | |
| # Cache the main Weaviate image (used by all compose files) | |
| - name: Cache Weaviate base image | |
| uses: actions/cache@v4 | |
| id: cache-weaviate-image | |
| with: | |
| path: /tmp/weaviate-base.tar | |
| key: ${{ runner.os }}-weaviate-image-1.32.0 | |
| restore-keys: | | |
| ${{ runner.os }}-weaviate-image- | |
| # Load or pull the main Weaviate image | |
| - name: Load or pull Weaviate base image | |
| run: | | |
| echo "=== Disk space before Weaviate image operations ===" | |
| df -h | |
| if [[ "${{ steps.cache-weaviate-image.outputs.cache-hit }}" == "true" ]]; then | |
| echo "✅ Cache hit. Loading Weaviate base image from cache." | |
| docker load --input /tmp/weaviate-base.tar | |
| else | |
| echo "❌ Cache miss. Pulling Weaviate base image and saving to cache." | |
| docker pull cr.weaviate.io/semitechnologies/weaviate:1.32.0 | |
| docker save cr.weaviate.io/semitechnologies/weaviate:1.32.0 -o /tmp/weaviate-base.tar | |
| fi | |
| echo "=== Disk space after Weaviate base image ===" | |
| df -h | |
| # Cache the multi2vec-clip model image | |
| - name: Cache multi2vec-clip image | |
| id: cache-clip-image | |
| uses: actions/cache@v4 | |
| with: | |
| path: /tmp/clip-image.tar | |
| key: ${{ runner.os }}-docker-image-multi2vec-clip-st-clip-vit-b-32 | |
| restore-keys: | | |
| ${{ runner.os }}-docker-image-multi2vec-clip- | |
| # Load or pull the multi2vec-clip image | |
| - name: Load or pull multi2vec-clip image | |
| run: | | |
| if [[ "${{ steps.cache-clip-image.outputs.cache-hit }}" == "true" ]]; then | |
| echo "✅ Cache hit. Loading multi2vec-clip image from cache." | |
| docker load --input /tmp/clip-image.tar | |
| else | |
| echo "❌ Cache miss. Pulling multi2vec-clip image and saving to cache." | |
| docker pull cr.weaviate.io/semitechnologies/multi2vec-clip:sentence-transformers-clip-ViT-B-32 | |
| docker save cr.weaviate.io/semitechnologies/multi2vec-clip:sentence-transformers-clip-ViT-B-32 -o /tmp/clip-image.tar | |
| fi | |
| # Cache the Ollama base image | |
| - name: Cache Ollama base image | |
| uses: actions/cache@v4 | |
| id: cache-ollama-image | |
| with: | |
| path: /tmp/ollama-base.tar | |
| key: ${{ runner.os }}-ollama-image-latest | |
| restore-keys: | | |
| ${{ runner.os }}-ollama-image- | |
| # Load or pull the Ollama base image | |
| - name: Load or pull Ollama base image | |
| run: | | |
| echo "=== Disk space before Ollama image operations ===" | |
| df -h | |
| if [[ "${{ steps.cache-ollama-image.outputs.cache-hit }}" == "true" ]]; then | |
| echo "✅ Cache hit. Loading Ollama base image from cache." | |
| docker load --input /tmp/ollama-base.tar | |
| else | |
| echo "❌ Cache miss. Pulling Ollama base image and saving to cache." | |
| docker pull ollama/ollama:latest | |
| docker save ollama/ollama:latest -o /tmp/ollama-base.tar | |
| fi | |
| echo "=== Disk space after Ollama base image ===" | |
| df -h | |
| # Cache Ollama models directory | |
| - name: Cache Ollama models | |
| uses: actions/cache@v4 | |
| id: cache-ollama-models | |
| with: | |
| path: ~/.ollama | |
| key: ${{ runner.os }}-ollama-models-snowflake-nomic-llama32-v1 | |
| restore-keys: | | |
| ${{ runner.os }}-ollama-models- | |
| # Prepare Ollama models directory | |
| - name: Prepare Ollama models directory | |
| run: | | |
| echo "=== Preparing Ollama models directory ===" | |
| mkdir -p ~/.ollama | |
| ls -la ~/.ollama || true | |
| if [[ "${{ steps.cache-ollama-models.outputs.cache-hit }}" == "true" ]]; then | |
| echo "✅ Ollama models cache hit. Models should be available." | |
| ls -la ~/.ollama | |
| else | |
| echo "❌ Ollama models cache miss. Models will be downloaded." | |
| fi | |
| # Start Docker containers (your existing step - MODIFIED) | |
| - name: Start Weaviate and Ollama containers | |
| run: | | |
| echo "Starting Docker containers..." | |
| ./tests/start-weaviate.sh | |
| echo "=== Container status ===" | |
| docker ps | |
| # MODIFIED - Smart Ollama model pulling | |
| - name: Pull Ollama Models (if not cached) | |
| run: | | |
| echo "Checking Ollama model availability..." | |
| sleep 5 # Wait for Ollama service to be ready | |
| models=("snowflake-arctic-embed" "nomic-embed-text" "llama3.2") | |
| for model in "${models[@]}"; do | |
| echo "Checking model: $model" | |
| if docker exec tests-ollama-1 ollama list | grep -q "$model"; then | |
| echo "✅ Model $model already available (from cache)" | |
| else | |
| echo "⬇️ Pulling model: $model" | |
| docker exec tests-ollama-1 ollama pull "$model" | |
| fi | |
| done | |
| echo "=== Final model list ===" | |
| docker exec tests-ollama-1 ollama list | |
| echo "✅ All models ready." | |
| # Wait for services to be ready | |
| - name: Wait for Weaviate services | |
| run: | | |
| echo "Checking if Weaviate services are ready..." | |
| # List of ports to check based on your docker-compose files | |
| ports=(8099 8580 8080 8090 8280 8180 8181 8182) | |
| for port in "${ports[@]}"; do | |
| echo "Checking port $port..." | |
| timeout=120 # 2 minutes timeout per service | |
| count=0 | |
| while [ $count -lt $timeout ]; do | |
| if curl -sf "http://localhost:$port/v1/.well-known/ready" >/dev/null 2>&1; then | |
| echo "✅ Service on port $port is ready" | |
| break | |
| fi | |
| if [ $((count % 10)) -eq 0 ]; then | |
| echo "⏳ Waiting for service on port $port... (${count}s elapsed)" | |
| fi | |
| sleep 1 | |
| count=$((count + 1)) | |
| done | |
| if [ $count -ge $timeout ]; then | |
| echo "⚠️ Service on port $port not ready after ${timeout}s, continuing..." | |
| fi | |
| done | |
| echo "Service readiness check completed" | |
| - name: Run tests | |
| env: | |
| DOCS_GITHUB_ENV: "true" # Tell conftest.py that GitHub Actions is managing Docker | |
| WEAVIATE_URL: ${{ secrets.WEAVIATE_URL }} | |
| WEAVIATE_HOST: ${{ secrets.WEAVIATE_HOST }} | |
| WEAVIATE_HOSTNAME: ${{ secrets.WEAVIATE_HOSTNAME }} | |
| WEAVIATE_HTTP_HOST: ${{ secrets.WEAVIATE_HTTP_HOST }} | |
| WEAVIATE_GRPC_HOST: ${{ secrets.WEAVIATE_GRPC_HOST }} | |
| WEAVIATE_API_KEY: ${{ secrets.WEAVIATE_API_KEY }} | |
| WEAVIATE_LOCAL_API_KEY: ${{ secrets.WEAVIATE_LOCAL_API_KEY }} | |
| COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} | |
| COHERE_APIKEY: ${{ secrets.COHERE_APIKEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| OPENAI_APIKEY: ${{ secrets.OPENAI_APIKEY }} | |
| HUGGINGFACE_APIKEY: ${{ secrets.HUGGINGFACE_APIKEY }} | |
| ANTHROPIC_APIKEY: ${{ secrets.ANTHROPIC_APIKEY }} | |
| WCD_USERNAME: ${{ secrets.WCD_USERNAME }} | |
| WCD_PASSWORD: ${{ secrets.WCD_PASSWORD }} | |
| run: | | |
| source .venv/bin/activate | |
| pytest -m pyv4 -v | |
| # Clean up containers | |
| - name: Stop Weaviate containers | |
| if: always() | |
| run: | | |
| echo "Stopping Weaviate Docker containers..." | |
| ./tests/stop-weaviate.sh || true | |
| - name: Final cleanup | |
| if: always() | |
| run: | | |
| echo "Final cleanup of Docker resources..." | |
| for file in tests/docker-compose*.yml; do | |
| if [ -f "$file" ]; then | |
| echo "Ensuring $file services are stopped" | |
| docker compose -f "$file" down -v || true | |
| fi | |
| done | |
| docker system prune -af || true |