Merge pull request #65 from raptorsun/hermetic

LCORE-791: konflux hermetic build

Author: tisnik

.tekton/rag-tool-pull-request.yaml

@@ -30,8 +30,42 @@ spec:
   - name: build-platforms
     value:
     - linux/x86_64
+    - linux-c6gd2xlarge/arm64
   - name: dockerfile
     value: Containerfile
+  - name: build-source-image
+    value: 'true'
+  - name: prefetch-input
+    # no source available: torch, faiss-cpu
+    # hermeto prefetch problems: uv, pip, jiter, tiktoken,
+    # those need cmake to build: pyarrow
+    # those need cargo to build: jiter, tiktoken, cryptography, fastuuid, hf_xet, maturin, pydantic_core, rpds_py, safetensors, tokenizers
+    # to accelerate build:numpy, scipy, pandas, pillow, scikit_learn
+    value: |
+     [
+        {
+          "type": "rpm",
+          "path": "."
+        },
+        {
+          "type": "pip",
+          "path": ".",
+          "requirements_files": [
+            "requirements.hashes.wheel.txt",
+            "requirements.hashes.source.txt",
+            "requirements.hermetic.txt"
+          ],
+          "requirements_build_files": ["requirements-build.txt"],
+          "binary": {
+            "packages": "accelerate,aiohappyeyeballs,aiohttp,aiosignal,aiosqlite,annotated-doc,annotated-types,anyio,asyncpg,attrs,beautifulsoup4,cffi,chardet,charset-normalizer,click,colorama,cryptography,dataclasses-json,defusedxml,distro,docling-ibm-models,einops,et-xmlfile,faiss-cpu,filetype,fire,frozenlist,googleapis-common-protos,greenlet,h11,hf-xet,httpcore,httpx,huggingface-hub,idna,jinja2,jiter,joblib,jsonlines,jsonref,jsonschema-specifications,latex2mathml,llama-stack-client,lxml,markdown-it-py,markupsafe,mdurl,mpire,mpmath,multidict,mypy-extensions,nest-asyncio,networkx,nltk,openpyxl,opentelemetry-api,opentelemetry-exporter-otlp-proto-common,opentelemetry-exporter-otlp-proto-http,opentelemetry-proto,opentelemetry-sdk,opentelemetry-semantic-conventions,packaging,pandas,pillow,platformdirs,pluggy,prompt-toolkit,propcache,psycopg2-binary,pyaml,pycparser,pydantic,pydantic-core,pydantic-settings,pygments,pyjwt,pylatexenc,python-dateutil,python-docx,python-dotenv,python-multipart,python-pptx,pytz,pyyaml,referencing,requests,rich,rpds-py,rtree,safetensors,scikit-learn,scipy,semchunk,sentence-transformers,shapely,shellingham,six,sniffio,sqlalchemy,starlette,sympy,tabulate,tenacity,threadpoolctl,tiktoken,tokenizers,torch,torchvision,tqdm,transformers,triton,typer,typing-extensions,typing-inspect,typing-inspection,tzdata,wcwidth,wrapt,xlsxwriter,yarl,zipp,uv-build,uv,pip,maturin,opencv-python,rapidocr,sqlite-vec",
+            "os": "linux",
+            "arch": "x86_64,aarch64",
+            "py_version": "312"
+          }
+        }
+      ]
+  - name: hermetic
+    value: 'true'
   pipelineSpec:
     description: |
       This pipeline is ideal for building multi-arch container images from a Containerfile while maintaining trust after pipeline customization.

.tekton/rag-tool-push.yaml

@@ -29,6 +29,39 @@ spec:
     - linux/x86_64
   - name: dockerfile
     value: Containerfile
+  - name: build-source-image
+    value: 'true'
+  - name: prefetch-input
+    # no source available: torch, faiss-cpu
+    # hermeto prefetch problems: uv, pip, jiter, tiktoken,
+    # those need cmake to build: pyarrow
+    # those need cargo to build: jiter, tiktoken, cryptography, fastuuid, hf_xet, maturin, pydantic_core, rpds_py, safetensors, tokenizers
+    # to accelerate build:numpy, scipy, pandas, pillow, scikit_learn
+    value: |
+     [
+        {
+          "type": "rpm",
+          "path": "."
+        },
+        {
+          "type": "pip",
+          "path": ".",
+          "requirements_files": [
+            "requirements.hashes.wheel.txt",
+            "requirements.hashes.source.txt",
+            "requirements.hermetic.txt"
+          ],
+          "requirements_build_files": ["requirements-build.txt"],
+          "binary": {
+            "packages": "accelerate,aiohappyeyeballs,aiohttp,aiosignal,aiosqlite,annotated-doc,annotated-types,anyio,asyncpg,attrs,beautifulsoup4,cffi,chardet,charset-normalizer,click,colorama,cryptography,dataclasses-json,defusedxml,distro,docling-ibm-models,einops,et-xmlfile,faiss-cpu,filetype,fire,frozenlist,googleapis-common-protos,greenlet,h11,hf-xet,httpcore,httpx,huggingface-hub,idna,jinja2,jiter,joblib,jsonlines,jsonref,jsonschema-specifications,latex2mathml,llama-stack-client,lxml,markdown-it-py,markupsafe,mdurl,mpire,mpmath,multidict,mypy-extensions,nest-asyncio,networkx,nltk,openpyxl,opentelemetry-api,opentelemetry-exporter-otlp-proto-common,opentelemetry-exporter-otlp-proto-http,opentelemetry-proto,opentelemetry-sdk,opentelemetry-semantic-conventions,packaging,pandas,pillow,platformdirs,pluggy,prompt-toolkit,propcache,psycopg2-binary,pyaml,pycparser,pydantic,pydantic-core,pydantic-settings,pygments,pyjwt,pylatexenc,python-dateutil,python-docx,python-dotenv,python-multipart,python-pptx,pytz,pyyaml,referencing,requests,rich,rpds-py,rtree,safetensors,scikit-learn,scipy,semchunk,sentence-transformers,shapely,shellingham,six,sniffio,sqlalchemy,starlette,sympy,tabulate,tenacity,threadpoolctl,tiktoken,tokenizers,torch,torchvision,tqdm,transformers,triton,typer,typing-extensions,typing-inspect,typing-inspection,tzdata,wcwidth,wrapt,xlsxwriter,yarl,zipp,uv-build,uv,pip,maturin,opencv-python,rapidocr,sqlite-vec",
+            "os": "linux",
+            "arch": "x86_64,aarch64",
+            "py_version": "312"
+          }
+        }
+      ]
+  - name: hermetic
+    value: 'true'
   pipelineSpec:
     description: |
       This pipeline is ideal for building multi-arch container images from a Containerfile while maintaining trust after pipeline customization.

Containerfile

@@ -9,8 +9,9 @@ RUN microdnf install -y --nodocs --setopt=keepcache=0 --setopt=tsflags=nodocs \
 RUN microdnf install -y rubygems && \
     microdnf clean all && \
     gem install asciidoctor
+
 # Install uv package manager
-RUN pip3.12 install uv==0.7.20
+RUN pip3.12 install uv>=0.7.20
 
 WORKDIR /rag-content
 
@@ -21,8 +22,11 @@ COPY scripts ./scripts
 
 # Configure UV environment variables for optimal performance
 # Pytorch backend - cpu. `uv` contains convenient way to specify the backend.
+# MATURIN_NO_INSTALL_RUST=1 : Disable installation of Rust dependencies by Maturin.
 ENV UV_COMPILE_BYTECODE=0 \
-    UV_PYTHON_DOWNLOADS=0
+    UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=0 \
+    MATURIN_NO_INSTALL_RUST=1
 
 # Install Python dependencies
 RUN uv sync --locked --no-install-project
@@ -43,4 +47,12 @@ RUN python ./scripts/download_embeddings_model.py \
 # Reset the entrypoint.
 ENTRYPOINT []
 
-LABEL description="Contains embedding model and dependencies needed to generate a vector database"
+LABEL vendor="Red Hat, Inc." \
+    name="lightspeed-core/rag-tool-rhel9" \
+    com.redhat.component="lightspeed-core/rag-tool" \
+    cpe="cpe:/a:redhat:lightspeed_core:0.4::el9" \
+    io.k8s.display-name="Lightspeed RAG Tool" \
+    summary="RAG tool containing embedding model and dependencies needed to generate a vector database." \
+    description="RAG Tool provides a shared codebase for generating vector databases. It serves as the core framework for Lightspeed-related projects (e.g., OpenShift Lightspeed, OpenStack Lightspeed, etc.) to generate their own vector databases that can be used for RAG." \
+    io.k8s.description="RAG Tool provides a shared codebase for generating vector databases. It serves as the core framework for Lightspeed-related projects (e.g., OpenShift Lightspeed, OpenStack Lightspeed, etc.) to generate their own vector databases that can be used for RAG." \
+    io.openshift.tags="lightspeed-core,lightspeed-rag-tool,lightspeed"

Makefile

@@ -101,6 +101,9 @@ start-postgres-debug: ## Start postgresql from the pgvector container image with
 	 -v ./postgresql/data:/var/lib/postgresql/data:Z pgvector/pgvector:pg16 \
 	 postgres -c log_statement=all -c log_destination=stderr
 
+konflux-requirements:	## generate hermetic requirements.*.txt file for konflux build
+	./scripts/konflux_requirements.sh
+
 .PHONY: help
 help: ## Show this help screen
 	@echo 'Usage: make <OPTIONS> ... <TARGETS>'

pyproject.toml

@@ -47,8 +47,8 @@ dependencies = [
     "llama-index-vector-stores-postgres>=0.5.4",
     # Pin torch/torchvision to versions available as CPU wheels
     # torch 2.5.x pairs with torchvision 0.20.x
-    "torch>=2.5.0,<2.6.0",
-    "torchvision>=0.20.0,<0.21.0",
+    "torch>=2.8.0,<2.9.0",
+    "torchvision>=0.23.0,<0.24.0",
     "llama-stack==0.3.5",
     "llama-stack-client==0.3.5",
     "aiosqlite>=0.21.0",
@@ -87,6 +87,8 @@ dev = [
     "pytest>=8.3.4",
     "pytest-cov>=6.0.0",
     "pytest-mock>=3.15.1",
+    "pybuild-deps>=0.5.0",
+    "pip==24.3.1",
 ]
 
 [tool.pylint."MESSAGES CONTROL"]

requirements-build.txt

@@ -0,0 +1,214 @@
+#
+# This file is autogenerated by pip-compile with Python 3.12
+# by the following command:
+#
+#    pybuild-deps compile --output-file=requirements-build.txt requirements.source.txt
+#
+altgraph==0.17.5
+    # via macholib
+bashlex==0.18
+    # via cibuildwheel
+bracex==2.6
+    # via cibuildwheel
+build==1.4.0
+    # via cibuildwheel
+calver==2025.10.20
+    # via trove-classifiers
+certifi==2026.1.4
+    # via cibuildwheel
+cibuildwheel==3.3.1
+    # via docling-parse
+cmake==3.31.10
+    # via docling-parse
+coherent-licensed==0.5.2
+    # via importlib-metadata
+ctypesgen @ git+https://github.com/pypdfium2-team/ctypesgen@b561360fad763b4a64e2d8ef8f7ddf354670dbb7
+    # via pypdfium2
+cython==3.2.4
+    # via
+    #   numpy
+    #   pyclipper
+delocate==0.13.0
+    # via docling-parse
+dependency-groups==1.3.1
+    # via cibuildwheel
+filelock==3.20.3
+    # via cibuildwheel
+flit-core==3.12.0
+    # via
+    #   build
+    #   coherent-licensed
+    #   dependency-groups
+    #   marshmallow
+    #   packaging
+    #   pathspec
+    #   pypdf
+    #   pyproject-hooks
+    #   pyproject-metadata
+    #   typing-extensions
+    #   wheel
+hatch-fancy-pypi-readme==25.1.0
+    # via
+    #   jsonschema
+    #   openai
+hatch-vcs==0.5.0
+    # via
+    #   filelock
+    #   fsspec
+    #   humanize
+    #   jsonschema
+    #   platformdirs
+    #   scikit-build-core
+    #   termcolor
+    #   urllib3
+hatchling==1.26.3
+    # via
+    #   hatch-fancy-pypi-readme
+    #   openai
+hatchling==1.28.0
+    # via
+    #   banks
+    #   bracex
+    #   cibuildwheel
+    #   filelock
+    #   fsspec
+    #   hatch-fancy-pypi-readme
+    #   hatch-vcs
+    #   humanize
+    #   jsonschema
+    #   llama-cloud-services
+    #   llama-index
+    #   llama-index-cli
+    #   llama-index-core
+    #   llama-index-embeddings-huggingface
+    #   llama-index-embeddings-openai
+    #   llama-index-indices-managed-llama-cloud
+    #   llama-index-instrumentation
+    #   llama-index-llms-openai
+    #   llama-index-readers-file
+    #   llama-index-readers-llama-parse
+    #   llama-index-vector-stores-faiss
+    #   llama-index-vector-stores-postgres
+    #   llama-parse
+    #   platformdirs
+    #   polyfactory
+    #   scikit-build-core
+    #   soupsieve
+    #   termcolor
+    #   urllib3
+    #   uvicorn
+humanize==4.15.0
+    # via cibuildwheel
+macholib==1.16.4
+    # via delocate
+maturin==1.10.2
+    # via uv-build
+meson-python==0.19.0
+    # via numpy
+meson==1.10.1
+    # via meson-python
+packaging==25.0
+    # via
+    #   cibuildwheel
+    #   hatchling
+    #   meson-python
+    #   pypdfium2
+    #   scikit-build-core
+    #   setuptools-scm
+patchelf==0.17.2.4
+    # via cibuildwheel
+pathspec==1.0.3
+    # via
+    #   hatchling
+    #   scikit-build-core
+pdm-backend==2.4.6
+    # via
+    #   fastapi
+    #   griffe
+    #   marko
+platformdirs==4.5.1
+    # via cibuildwheel
+pluggy==1.6.0
+    # via hatchling
+poetry-core==2.3.0
+    # via
+    #   llama-cloud
+    #   tomlkit
+pybind11==3.0.1
+    # via docling-parse
+pyelftools==0.32
+    # via cibuildwheel
+pyproject-hooks==1.2.0
+    # via build
+pyproject-metadata==0.10.0
+    # via meson-python
+scikit-build-core==0.11.6
+    # via
+    #   cmake
+    #   patchelf
+    #   pybind11
+semantic-version==2.10.0
+    # via setuptools-rust
+setuptools-rust==1.12.0
+    # via maturin
+setuptools-scm==9.2.2
+    # via
+    #   ctypesgen
+    #   delocate
+    #   hatch-vcs
+    #   importlib-metadata
+    #   pluggy
+    #   pyclipper
+    #   setuptools-rust
+    #   urllib3
+trove-classifiers==2026.1.14.14
+    # via hatchling
+typing-extensions==4.15.0
+    # via delocate
+uv-build==0.9.26
+    # via llama-index-workflows
+wheel==0.45.1
+    # via
+    #   bashlex
+    #   cibuildwheel
+    #   docling-parse
+    #   meson
+    #   pyclipper
+    #   pypdfium2
+    #   tree-sitter-c
+    #   tree-sitter-javascript
+    #   tree-sitter-python
+    #   tree-sitter-typescript
+
+# The following packages are considered to be unsafe in a requirements file:
+setuptools==80.10.1
+    # via
+    #   bashlex
+    #   calver
+    #   certifi
+    #   colorlog
+    #   ctypesgen
+    #   delocate
+    #   dill
+    #   docling-parse
+    #   importlib-metadata
+    #   llama-stack
+    #   maturin
+    #   meson
+    #   multiprocess
+    #   pathspec
+    #   pgvector
+    #   pluggy
+    #   psutil
+    #   pyclipper
+    #   pyelftools
+    #   pypdfium2
+    #   regex
+    #   setuptools-rust
+    #   setuptools-scm
+    #   tree-sitter
+    #   tree-sitter-c
+    #   tree-sitter-javascript
+    #   tree-sitter-python
+    #   tree-sitter-typescript
+    #   trove-classifiers