Skip to content

Commit 7c0f774

Browse files
carlocamillonicarlocamilloni
authored
Merge pull request #1372 from Iximiel/feature/pythonCI
Future-proofing the CI against future Python versions
2 parents ac02bfe + 13c1e8d commit 7c0f774

34 files changed

+55598
-504
lines changed

‎.ci/pytorch/requirements.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# The extra url makes pip grap the torch fersion we want to use in the regtests
2+
--extra-index-url=https://download.pytorch.org/whl/cpu
3+
torch>=2.7
4+
metatomic-torch>=0.1.3,<0.2
5+
featomic-torch==0.7.0

‎.ci/requirements-codecheck.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Cython==3.2.4

‎.github/actions/linux-setup/action.yaml‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
name: 'Prepare linux environment'
2+
description: 'Set up the linux environment for plumed'
3+
inputs:
4+
pytest-make:
5+
descriptions: 'set to yes for exporting PYTHONPATH to be used in the tests'
6+
required: false
7+
default: false
8+
type: boolean
9+
export-kernel:
10+
descriptions: 'set to yes for exporting PLUMED_KERNEL to be used in the tests'
11+
required: false
12+
default: false
13+
type: boolean
14+
mpi:
15+
description: 'setup the mpi library and environmental variables'
16+
required: false
17+
default: false
18+
type: boolean
19+
ccache:
20+
description: 'setup ccache and the environmentalvariable to use che chache action with the ~/.ccache directory'
21+
required: false
22+
default: false
23+
type: boolean
24+
boost-debug:
25+
description: 'install boost libraries in debug mode'
26+
required: false
27+
default: false
28+
type: boolean
29+
runs:
30+
using: "composite"
31+
steps:
32+
# the boolean are evaluated to string... -> the various "=='true'"
33+
- name: Remove unused stuff
34+
run: |
35+
echo "Making some space"
36+
df -h
37+
sudo rm -rf /usr/share/dotnet
38+
sudo rm -rf /usr/local/lib/android
39+
sudo rm -rf /opt/ghc
40+
df -h
41+
shell: bash
42+
43+
- name: Install generic packages
44+
run: |
45+
echo "Install generic packages"
46+
sudo apt-get update -qq
47+
sudo apt-get install -y libatlas-base-dev
48+
sudo apt-get install -y libfftw3-dev
49+
sudo apt-get install -y gsl-bin
50+
sudo apt-get install -y libgsl0-dev
51+
shell: bash
52+
53+
- name: Install boost
54+
if: ${{ inputs.boost-debug != 'true' }}
55+
run: |
56+
echo "Install boost"
57+
sudo apt-get install -y libboost-serialization-dev
58+
shell: bash
59+
60+
- name: Install boost in debug mode
61+
if: ${{ inputs.boost-debug == 'true' }}
62+
run: |
63+
echo "Install boost in debug mode"
64+
${{ github.action_path }}/install.boost
65+
shell: bash
66+
67+
- name: Set paths for the installed plumed
68+
run: |
69+
echo "Setting up the basig enviroment variables for running plumed installed with --prefix=\"\$HOME/opt\""
70+
#accessing installed plumed
71+
echo "$HOME/opt/bin" >> $GITHUB_PATH
72+
echo "CPATH=$HOME/opt/include:$CPATH" >> $GITHUB_ENV
73+
echo "INCLUDE=$HOME/opt/include:$INCLUDE" >> $GITHUB_ENV
74+
echo "LIBRARY_PATH=$HOME/opt/lib:$LIBRARY_PATH" >> $GITHUB_ENV
75+
echo "LD_LIBRARY_PATH=$HOME/opt/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
76+
shell: bash
77+
78+
- name: Set paths for python tests
79+
if: ${{ inputs.pytest-make == 'true' }}
80+
run: |
81+
echo "Preparing some variables for pytest"
82+
# path required for pytest:
83+
echo "$HOME/.local/bin" >> $GITHUB_PATH
84+
echo "PYTHONPATH=$HOME/opt/lib/plumed/python:$PYTHONPATH" >> $GITHUB_ENV
85+
shell: bash
86+
87+
- name: Export PLUMED_KERNEL
88+
if: ${{ inputs.export-kernel == 'true' }}
89+
run: |
90+
echo "Setting up PLUMED_KERNEL"
91+
echo "PLUMED_KERNEL=$HOME/opt/lib/libplumedKernel.so" >> $GITHUB_ENV
92+
shell: bash
93+
94+
- name: Prepare the environment with for mpi
95+
if: ${{ inputs.mpi == 'true' }}
96+
run: |
97+
echo "Setup opempi"
98+
sudo apt-get install -y libopenmpi-dev openmpi-bin
99+
echo "CC=mpicc" >> $GITHUB_ENV
100+
echo "CXX=mpic++" >> $GITHUB_ENV
101+
echo "OMPI_MCA_btl_base_warn_component_unused=0" >> $GITHUB_ENV
102+
echo "OMPI_MCA_btl_base_verbose=0" >> $GITHUB_ENV
103+
echo "OMPI_MCA_plm=isolated" >> $GITHUB_ENV
104+
echo "OMPI_MCA_btl_vader_single_copy_mechanism=none" >> $GITHUB_ENV
105+
echo "OMPI_MCA_rmaps_base_oversubscribe=yes" >> $GITHUB_ENV
106+
shell: bash
107+
108+
- name: Prepare the enviroment with for ccache
109+
if: ${{ inputs.ccache == 'true' }}
110+
run: |
111+
echo "Setup ccache"
112+
sudo apt-get install -y ccache
113+
#forcing the legacy directory for ccache
114+
echo "CCACHE_DIR=$HOME/.ccache/" >> $GITHUB_ENV
115+
shell: bash

.ci/install.boost renamed to .github/actions/linux-setup/install.boost

File renamed without changes.

‎.github/workflows/ci.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,11 @@ jobs:
2626
mac:
2727
uses: ./.github/workflows/macWF.yml
2828
secrets: inherit
29-
29+
30+
python:
31+
uses: ./.github/workflows/pythonWF.yml
32+
secrets: inherit
33+
3034
docker:
3135
uses: ./.github/workflows/dockerWF.yml
3236
secrets: inherit

‎.github/workflows/codecheckWF.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,13 @@ jobs:
2222
- name: Set path
2323
run: |
2424
echo "$HOME/opt/bin" >> $GITHUB_PATH
25+
- name: Set up Python for Cython
26+
uses: actions/setup-python@v6
27+
with:
28+
python-version: "3.12"
29+
cache: "pip"
30+
pip-install: -r .ci/requirements-codecheck.txt
31+
cache-dependency-path: .ci/requirements-codecheck.txt
2532
- name: Install requirements
2633
run: |
2734
.ci/install.cppcheck 2.13.4

‎.github/workflows/linuxWF.yml‎

Lines changed: 17 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -37,59 +37,25 @@ jobs:
3737
path: ~/.ccache
3838
key: ccache-reset1-linux${{ matrix.variant }}hash-${{ github.sha }}
3939
restore-keys: ccache-reset1-linux${{ matrix.variant }}hash-
40-
- name: Removed unused stuff
41-
run: |
42-
df -h
43-
sudo rm -rf /usr/share/dotnet
44-
sudo rm -rf /usr/local/lib/android
45-
sudo rm -rf /opt/ghc
46-
df -h
40+
- name: setup the CI environment
41+
uses: ./.github/actions/linux-setup
42+
with:
43+
ccache: true
44+
pytest-make: true
45+
boost-debug: ${{ contains( matrix.variant, '-debug-' ) }}
46+
mpi: ${{ contains( matrix.variant, '-mpi-' ) }}
47+
4748
- name: Set up Python
4849
uses: actions/setup-python@v5
4950
with:
50-
python-version: 3.9
51-
- name: Set paths
52-
run: |
53-
echo "$HOME/opt/bin" >> $GITHUB_PATH
54-
# path required for pytest:
55-
echo "$HOME/.local/bin" >> $GITHUB_PATH
56-
echo "PYTHONPATH=$HOME/opt/lib/plumed/python:$PYTHONPATH" >> $GITHUB_ENV
57-
echo "CPATH=$HOME/opt/include:$CPATH" >> $GITHUB_ENV
58-
echo "INCLUDE=$HOME/opt/include:$INCLUDE" >> $GITHUB_ENV
59-
echo "LIBRARY_PATH=$HOME/opt/lib:$LIBRARY_PATH" >> $GITHUB_ENV
60-
echo "LD_LIBRARY_PATH=$HOME/opt/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
61-
- name: Install generic packages
62-
run: |
63-
sudo apt-get update -qq
64-
sudo apt-get install -y libatlas-base-dev
65-
sudo apt-get install -y libfftw3-dev
66-
sudo apt-get install -y gsl-bin
67-
sudo apt-get install -y libgsl0-dev
68-
sudo apt-get install -y ccache
69-
- name: Install system boost
70-
if: ${{ ! contains( matrix.variant, '-debug-' ) }}
71-
run: |
72-
sudo apt-get install -y libboost-serialization-dev
73-
- name: Install boost with debug flags
74-
if: contains( matrix.variant, '-debug-' )
75-
run: |
76-
.ci/install.boost
51+
# testing the oldest supported python version, see the "python" workflow for newer version
52+
python-version: "3.10"
7753
- name: Install python packages needed for tests
7854
run: |
7955
python -m pip install --upgrade pip
80-
pip install --user --extra-index-url=https://download.pytorch.org/whl/cpu \
81-
Cython \
82-
numpy \
83-
pytest \
84-
six \
85-
pandas \
86-
mdtraj \
87-
MDAnalysis \
88-
"torch>=2.7" \
89-
"metatomic-torch>=0.1.3,<0.2" \
90-
"featomic-torch==0.7.0"
91-
92-
# torch 2.7 above is the first one to use cxx11 ABI for the PyPI wheels
56+
pip install --user -r ./python/requirements_build.txt
57+
pip install --user -r ./python/requirements_run.txt
58+
pip install --user -r ./.ci/pytorch/requirements.txt
9359
9460
- name: Configure libmetatomic & libtorch
9561
if: ${{ ! contains( matrix.variant, '-debug-' ) }} # the libraries on PyPI are not compiled with GLIBCXX_DEBUG
@@ -102,14 +68,11 @@ jobs:
10268
10369
# the flags above enable the use of both libmetatomic and libtorch
10470
echo "PLUMED_CONFIG=$PLUMED_CONFIG --enable-libmetatomic --enable-libtorch" >> $GITHUB_ENV
105-
- name: Install Doxygen
71+
- name: Install tools for setting up the manuals
10672
if: contains( matrix.variant, '-doc-' )
10773
run: |
108-
pip install requests
109-
pip install PlumedToHTML
110-
pip install networkx
111-
pip install mkdocs-material
112-
pip install python-markdown-math
74+
python -m pip install --upgrade pip
75+
pip install --user -r ./new-manual/requirements.txt
11376
sudo apt-get install -y graphviz
11477
sudo apt-get install -y doxygen-latex
11578
.ci/install.doxygen Release_1_13_2
@@ -150,14 +113,6 @@ jobs:
150113
# install MPI at last since it modifies CC and CXX
151114
if: contains( matrix.variant, '-mpi-' )
152115
run: |
153-
sudo apt-get install -y libopenmpi-dev openmpi-bin
154-
echo "CC=mpicc" >> $GITHUB_ENV
155-
echo "CXX=mpic++" >> $GITHUB_ENV
156-
echo "OMPI_MCA_btl_base_warn_component_unused=0" >> $GITHUB_ENV
157-
echo "OMPI_MCA_btl_base_verbose=0" >> $GITHUB_ENV
158-
echo "OMPI_MCA_plm=isolated" >> $GITHUB_ENV
159-
echo "OMPI_MCA_btl_vader_single_copy_mechanism=none" >> $GITHUB_ENV
160-
echo "OMPI_MCA_rmaps_base_oversubscribe=yes" >> $GITHUB_ENV
161116
pip install --user mpi4py
162117
python -c "import mpi4py"
163118
- name: Configure PLUMED
@@ -230,6 +185,7 @@ jobs:
230185
- name: Run python tests
231186
run: |
232187
cd python
188+
pip install --user pytest
233189
pytest -v
234190
- name: Build doc
235191
env:

‎.github/workflows/pythonWF.yml‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
name: python tests
2+
3+
on:
4+
workflow_call:
5+
6+
env:
7+
# on CI, better dump stack trace in case there is an error
8+
PLUMED_STACK_TRACE: yes
9+
# use two threads for openMP tests
10+
PLUMED_NUM_THREADS: 2
11+
# these are used to build required packages
12+
CC: gcc
13+
CXX: g++
14+
15+
jobs:
16+
linux-build-plumed:
17+
runs-on: ubuntu-22.04
18+
steps:
19+
- uses: actions/checkout@v4
20+
- uses: actions/cache@v4
21+
with:
22+
path: ~/.ccache
23+
key: ccache-python-linux-${{ github.sha }}
24+
restore-keys: ccache-python-linux-
25+
- name: setup the CI environment
26+
uses: ./.github/actions/linux-setup
27+
with:
28+
ccache: true
29+
30+
- name: Build PLUMED
31+
run: |
32+
ccache -s -M 100M
33+
./configure CXX="ccache $CXX" --enable-boost_serialization --disable-dependency-tracking --enable-modules=all $PLUMED_CONFIG --prefix=$HOME/opt
34+
make -j 4
35+
make install
36+
# check for global symbols, see https://github.com/plumed/plumed2/issues/549
37+
make nmcheck
38+
ccache -s -M 100M
39+
- name: prepare the plumedArtifact
40+
run: |
41+
cd $HOME
42+
tar cf plumed.tar opt/
43+
44+
- name: Upload the Plumed artifact
45+
uses: actions/upload-artifact@v6
46+
with:
47+
name: plumed-python-linux
48+
path: ~/plumed.tar
49+
retention-days: 1
50+
51+
linux-test-python:
52+
needs:
53+
- linux-build-plumed
54+
runs-on: ubuntu-22.04
55+
strategy:
56+
fail-fast: false
57+
matrix:
58+
pyversion: [ "3.9", "3.10", "3.11", "3.12", "3.13" ]
59+
steps:
60+
#still checking out to install the plumed package and pycv
61+
- uses: actions/checkout@v4
62+
- name: Set up Python
63+
uses: actions/setup-python@v6
64+
with:
65+
python-version: ${{ matrix.pyversion }}
66+
pip-install: pytest -r ./python/requirements_run.txt
67+
68+
- name: setup the CI environment
69+
uses: ./.github/actions/linux-setup
70+
with:
71+
export-kernel: true
72+
73+
- name: Download artifacts
74+
uses: actions/download-artifact@v7
75+
with:
76+
path: ~/
77+
name: plumed-python-linux
78+
79+
- name: Install plumed
80+
run: |
81+
cd $HOME
82+
tar xf ~/plumed.tar
83+
84+
- name: Install the python pacakge
85+
working-directory: ./python
86+
run: |
87+
make PLUMED_VERSION
88+
pip install .
89+
- name: Run python tests
90+
working-directory: ./python
91+
run: |
92+
pytest -v
93+
- name: Compile and test pycv
94+
working-directory: ./plugins/pycv/
95+
run: |
96+
ln -s $(realpath ../../regtest/scripts) ./regtest/scripts
97+
make check

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,6 @@ makefile.dep
2525
/autom4*
2626
/stamp-h
2727
__pycache__
28+
#to not export personal pyenv settings
29+
.python-version
30+

‎Makefile.conf.in‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,7 @@ datadir=@datadir@
3838
docdir=@docdir@
3939
htmldir=@htmldir@
4040
python_bin=@PYTHON_BIN@
41-
plumed_found_cython=@CYTHON_FOUND@
42-
canPyCV=@PLUMED_CAN_PYCV@
43-
pybind11_cflags=@PYBIND11_CFLAGS@
44-
python_cf_embedded=@PYTHON_CFLAGS@
45-
python_ld_embedded=@PYTHON_LDFLAGS@
41+
plumed_found_python_build=@PYTHON_BUILD_FOUND@
4642
mpiexec=@MPIEXEC@
4743
make_static_archive=@make_static_archive@
4844
use_absolute_soname=@use_absolute_soname@

0 commit comments

Comments
 (0)