Future-proofing the CI against future Python versions

.ci/pytorch/requirements.txt

@@ -0,0 +1,5 @@
+# The extra url makes pip grap the torch fersion we want to use in the regtests
+--extra-index-url=https://download.pytorch.org/whl/cpu
+torch>=2.7
+metatomic-torch>=0.1.3,<0.2
+featomic-torch==0.7.0

.ci/requirements-codecheck.txt

@@ -0,0 +1 @@
+Cython==3.2.4

.github/actions/linux-setup/action.yaml

@@ -0,0 +1,115 @@
+name: 'Prepare linux environment'
+description: 'Set up the linux environment for plumed'
+inputs:
+  pytest-make:
+    descriptions: 'set to yes for exporting PYTHONPATH to be used in the tests'
+    required: false
+    default: false
+    type: boolean
+  export-kernel:
+    descriptions: 'set to yes for exporting PLUMED_KERNEL to be used in the tests'
+    required: false
+    default: false
+    type: boolean
+  mpi:
+    description: 'setup the mpi library and environmental variables'
+    required: false
+    default: false
+    type: boolean
+  ccache:
+    description: 'setup ccache and the environmentalvariable to use che chache action with the ~/.ccache directory'
+    required: false
+    default: false
+    type: boolean
+  boost-debug:
+    description: 'install boost libraries in debug mode'
+    required: false
+    default: false
+    type: boolean
+runs:
+  using: "composite"
+  steps:
+    # the boolean are evaluated to string... -> the various "=='true'"
+    - name: Remove unused stuff
+      run: |
+        echo "Making some space"
+        df -h
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf /usr/local/lib/android
+        sudo rm -rf /opt/ghc
+        df -h
+      shell: bash
+
+    - name: Install generic packages
+      run: |
+        echo "Install generic packages"
+        sudo apt-get update -qq
+        sudo apt-get install -y libatlas-base-dev
+        sudo apt-get install -y libfftw3-dev
+        sudo apt-get install -y gsl-bin
+        sudo apt-get install -y libgsl0-dev
+      shell: bash
+
+    - name: Install boost
+      if: ${{ inputs.boost-debug != 'true' }}
+      run: |
+        echo "Install boost"
+        sudo apt-get install -y libboost-serialization-dev
+      shell: bash
+
+    - name: Install boost in debug mode
+      if: ${{ inputs.boost-debug == 'true' }}
+      run: |
+        echo "Install boost in debug mode"
+        ${{ github.action_path }}/install.boost
+      shell: bash
+
+    - name: Set paths for the installed plumed
+      run: |
+        echo "Setting up the basig enviroment variables for running plumed installed with --prefix=\"\$HOME/opt\""
+        #accessing installed plumed
+        echo "$HOME/opt/bin" >> $GITHUB_PATH
+        echo "CPATH=$HOME/opt/include:$CPATH" >> $GITHUB_ENV
+        echo "INCLUDE=$HOME/opt/include:$INCLUDE" >> $GITHUB_ENV
+        echo "LIBRARY_PATH=$HOME/opt/lib:$LIBRARY_PATH" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=$HOME/opt/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+      shell: bash
+
+    - name: Set paths for python tests
+      if: ${{ inputs.pytest-make  == 'true' }}
+      run: |
+        echo "Preparing some variables for pytest"
+        # path required for pytest:
+        echo "$HOME/.local/bin" >> $GITHUB_PATH
+        echo "PYTHONPATH=$HOME/opt/lib/plumed/python:$PYTHONPATH" >> $GITHUB_ENV
+      shell: bash
+
+    - name: Export PLUMED_KERNEL
+      if: ${{ inputs.export-kernel  == 'true' }}
+      run: |
+        echo "Setting up PLUMED_KERNEL"
+        echo "PLUMED_KERNEL=$HOME/opt/lib/libplumedKernel.so" >> $GITHUB_ENV
+      shell: bash
+
+    - name: Prepare the environment with for mpi
+      if: ${{ inputs.mpi  == 'true' }}
+      run: |
+        echo "Setup opempi"
+        sudo apt-get install -y libopenmpi-dev openmpi-bin
+        echo "CC=mpicc" >> $GITHUB_ENV
+        echo "CXX=mpic++" >> $GITHUB_ENV
+        echo "OMPI_MCA_btl_base_warn_component_unused=0" >> $GITHUB_ENV
+        echo "OMPI_MCA_btl_base_verbose=0" >> $GITHUB_ENV
+        echo "OMPI_MCA_plm=isolated" >> $GITHUB_ENV
+        echo "OMPI_MCA_btl_vader_single_copy_mechanism=none" >> $GITHUB_ENV
+        echo "OMPI_MCA_rmaps_base_oversubscribe=yes" >> $GITHUB_ENV
+      shell: bash
+
+    - name: Prepare the enviroment with for ccache
+      if: ${{ inputs.ccache  == 'true' }}
+      run: |
+        echo "Setup ccache"
+        sudo apt-get install -y ccache
+        #forcing the legacy directory for ccache
+        echo "CCACHE_DIR=$HOME/.ccache/" >> $GITHUB_ENV
+      shell: bash

.github/workflows/ci.yml

@@ -26,7 +26,11 @@ jobs:
   mac:
     uses: ./.github/workflows/macWF.yml
     secrets: inherit
-
+
+  python:
+    uses: ./.github/workflows/pythonWF.yml
+    secrets: inherit
+
   docker:
     uses: ./.github/workflows/dockerWF.yml
     secrets: inherit

.github/workflows/codecheckWF.yml

@@ -22,6 +22,13 @@ jobs:
     - name: Set path
       run: |
          echo "$HOME/opt/bin" >> $GITHUB_PATH
+    - name: Set up Python for Cython
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.12"
+        cache: "pip"
+        pip-install: -r .ci/requirements-codecheck.txt
+        cache-dependency-path: .ci/requirements-codecheck.txt
     - name: Install requirements
       run: |
         .ci/install.cppcheck 2.13.4

.github/workflows/linuxWF.yml

@@ -37,59 +37,25 @@ jobs:
         path: ~/.ccache
         key: ccache-reset1-linux${{ matrix.variant }}hash-${{ github.sha }}
         restore-keys: ccache-reset1-linux${{ matrix.variant }}hash-
-    - name: Removed unused stuff
-      run: |
-        df -h
-        sudo rm -rf /usr/share/dotnet
-        sudo rm -rf /usr/local/lib/android
-        sudo rm -rf /opt/ghc
-        df -h
+    - name: setup the CI environment
+      uses: ./.github/actions/linux-setup
+      with:
+        ccache: true
+        pytest-make: true
+        boost-debug: ${{ contains( matrix.variant, '-debug-' ) }}
+        mpi: ${{ contains( matrix.variant, '-mpi-' ) }}
+
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: 3.9
-    - name: Set paths
-      run: |
-        echo "$HOME/opt/bin" >> $GITHUB_PATH
-        # path required for pytest:
-        echo "$HOME/.local/bin" >> $GITHUB_PATH
-        echo "PYTHONPATH=$HOME/opt/lib/plumed/python:$PYTHONPATH" >> $GITHUB_ENV
-        echo "CPATH=$HOME/opt/include:$CPATH" >> $GITHUB_ENV
-        echo "INCLUDE=$HOME/opt/include:$INCLUDE" >> $GITHUB_ENV
-        echo "LIBRARY_PATH=$HOME/opt/lib:$LIBRARY_PATH" >> $GITHUB_ENV
-        echo "LD_LIBRARY_PATH=$HOME/opt/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
-    - name: Install generic packages
-      run: |
-        sudo apt-get update -qq
-        sudo apt-get install -y libatlas-base-dev
-        sudo apt-get install -y libfftw3-dev
-        sudo apt-get install -y gsl-bin
-        sudo apt-get install -y libgsl0-dev
-        sudo apt-get install -y ccache
-    - name: Install system boost
-      if: ${{ ! contains( matrix.variant, '-debug-' ) }}
-      run: |
-        sudo apt-get install -y libboost-serialization-dev
-    - name: Install boost with debug flags
-      if: contains( matrix.variant, '-debug-' )
-      run: |
-        .ci/install.boost
+        # testing the oldest supported python version, see the "python" workflow for newer version
+        python-version: "3.10"
     - name: Install python packages needed for tests
       run: |
         python -m pip install --upgrade pip
-        pip install --user --extra-index-url=https://download.pytorch.org/whl/cpu \
-              Cython \
-              numpy \
-              pytest \
-              six \
-              pandas \
-              mdtraj \
-              MDAnalysis \
-              "torch>=2.7" \
-              "metatomic-torch>=0.1.3,<0.2" \
-              "featomic-torch==0.7.0"
-
-        # torch 2.7 above is the first one to use cxx11 ABI for the PyPI wheels
+        pip install --user -r ./python/requirements_build.txt
+        pip install --user -r ./python/requirements_run.txt
+        pip install --user -r ./.ci/pytorch/requirements.txt
 
     - name: Configure libmetatomic & libtorch
       if: ${{ ! contains( matrix.variant, '-debug-' ) }} # the libraries on PyPI are not compiled with GLIBCXX_DEBUG
@@ -102,14 +68,11 @@ jobs:
 
         # the flags above enable the use of both libmetatomic and libtorch
         echo "PLUMED_CONFIG=$PLUMED_CONFIG --enable-libmetatomic --enable-libtorch" >> $GITHUB_ENV
-    - name: Install Doxygen
+    - name: Install tools for setting up the manuals
       if: contains( matrix.variant, '-doc-' )
       run: |
-        pip install requests
-        pip install PlumedToHTML
-        pip install networkx
-        pip install mkdocs-material
-        pip install python-markdown-math
+        python -m pip install --upgrade pip
+        pip install --user -r ./new-manual/requirements.txt
         sudo apt-get install -y graphviz
         sudo apt-get install -y doxygen-latex
         .ci/install.doxygen Release_1_13_2
@@ -150,14 +113,6 @@ jobs:
       # install MPI at last since it modifies CC and CXX
       if: contains( matrix.variant, '-mpi-' )
       run: |
-        sudo apt-get install -y libopenmpi-dev openmpi-bin
-        echo "CC=mpicc" >> $GITHUB_ENV
-        echo "CXX=mpic++" >> $GITHUB_ENV
-        echo "OMPI_MCA_btl_base_warn_component_unused=0" >> $GITHUB_ENV
-        echo "OMPI_MCA_btl_base_verbose=0" >> $GITHUB_ENV
-        echo "OMPI_MCA_plm=isolated" >> $GITHUB_ENV
-        echo "OMPI_MCA_btl_vader_single_copy_mechanism=none" >> $GITHUB_ENV
-        echo "OMPI_MCA_rmaps_base_oversubscribe=yes" >> $GITHUB_ENV
         pip install --user mpi4py
         python -c "import mpi4py"
     - name: Configure PLUMED
@@ -230,6 +185,7 @@ jobs:
     - name: Run python tests
       run: |
          cd python
+         pip install --user pytest
          pytest -v
     - name: Build doc
       env:

.github/workflows/pythonWF.yml

@@ -0,0 +1,97 @@
+name: python tests
+
+on:
+  workflow_call:
+
+env:
+# on CI, better dump stack trace in case there is an error
+  PLUMED_STACK_TRACE: yes
+# use two threads for openMP tests
+  PLUMED_NUM_THREADS: 2
+# these are used to build required packages
+  CC: gcc
+  CXX: g++
+
+jobs:
+  linux-build-plumed:
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/cache@v4
+      with:
+        path: ~/.ccache
+        key: ccache-python-linux-${{ github.sha }}
+        restore-keys: ccache-python-linux-
+    - name: setup the CI environment
+      uses: ./.github/actions/linux-setup
+      with:
+        ccache: true
+
+    - name: Build PLUMED
+      run: |
+        ccache -s -M 100M
+        ./configure CXX="ccache $CXX" --enable-boost_serialization --disable-dependency-tracking --enable-modules=all $PLUMED_CONFIG --prefix=$HOME/opt
+        make -j 4
+        make install
+        # check for global symbols, see https://github.com/plumed/plumed2/issues/549
+        make nmcheck
+        ccache -s -M 100M
+    - name: prepare the plumedArtifact
+      run: |
+        cd $HOME
+        tar cf plumed.tar opt/
+
+    - name: Upload the Plumed artifact
+      uses: actions/upload-artifact@v6
+      with:
+        name: plumed-python-linux
+        path: ~/plumed.tar
+        retention-days: 1
+
+  linux-test-python:
+    needs:
+    - linux-build-plumed
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        pyversion: [ "3.9", "3.10", "3.11", "3.12", "3.13" ]
+    steps:
+      #still checking out to install the plumed package and pycv
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.pyversion }}
+        pip-install: pytest -r ./python/requirements_run.txt
+
+    - name: setup the CI environment
+      uses: ./.github/actions/linux-setup
+      with:
+        export-kernel: true
+
+    - name: Download artifacts
+      uses: actions/download-artifact@v7
+      with:
+        path: ~/
+        name: plumed-python-linux
+
+    - name: Install plumed
+      run: |
+          cd $HOME
+          tar xf ~/plumed.tar
+
+    - name: Install the python pacakge
+      working-directory: ./python
+      run: |
+         make PLUMED_VERSION
+         pip install .
+    - name: Run python tests
+      working-directory: ./python
+      run: |
+         pytest -v
+    - name: Compile and test pycv
+      working-directory: ./plugins/pycv/
+      run: |
+            ln -s $(realpath ../../regtest/scripts) ./regtest/scripts
+            make check

.gitignore

@@ -25,3 +25,6 @@ makefile.dep
 /autom4*
 /stamp-h
 __pycache__
+#to not export personal pyenv settings
+.python-version
+

Makefile.conf.in

@@ -38,11 +38,7 @@ datadir=@datadir@
 docdir=@docdir@
 htmldir=@htmldir@
 python_bin=@PYTHON_BIN@
-plumed_found_cython=@CYTHON_FOUND@
-canPyCV=@PLUMED_CAN_PYCV@
-pybind11_cflags=@PYBIND11_CFLAGS@
-python_cf_embedded=@PYTHON_CFLAGS@
-python_ld_embedded=@PYTHON_LDFLAGS@
+plumed_found_python_build=@PYTHON_BUILD_FOUND@
 mpiexec=@MPIEXEC@
 make_static_archive=@make_static_archive@
 use_absolute_soname=@use_absolute_soname@

conda/plumed/build.sh

@@ -33,7 +33,7 @@ export LIBS="-lfftw3 -lgsl -lgslcblas -llapack -lblas -lz $LIBS"
 # --disable-libsearch forces to link only explicitely requested libraries
 # --disable-static-patch avoid tests that are only required for static patches
 # --disable-static-archive makes package smaller
-# --disable-basic-warnings deactivet the warning, compiling with the current compiler version (today is 28/10/2025) shows some false positives in ActionPilot
+# --disable-basic-warnings deactive the warning, compiling with the current compiler version (today is 28/10/2025) shows some false positives in ActionPilot
 ./configure --prefix=$PREFIX --disable-python --disable-libsearch --disable-static-patch --disable-static-archive --disable-basic-warnings
 
 make -j${CPU_COUNT}

conda/py-plumed/meta.yaml

@@ -19,7 +19,6 @@ requirements:
   host:
     - python
     - pip
-    - cython
   run:
     - plumed
     - python