Merge branch 'main' into paper

Author: jan-janssen

.ci_support/build_notebooks.sh

@@ -9,8 +9,8 @@ dependencies:
 - openmpi
 - cloudpickle =3.1.1
 - mpi4py =4.0.1
-- pyzmq =26.2.1
+- pyzmq =26.3.0
 - flux-core
 - jupyter-book =1.0.0
-- h5py =3.12.1
+- h5py =3.13.0
 - python =3.12

.ci_support/environment-docs.yml

@@ -0,0 +1,21 @@
+channels:
+- conda-forge
+dependencies:
+- jupyter
+- papermill
+- numpy
+- openmpi
+- cloudpickle =3.1.1
+- mpi4py
+- pyzmq =26.3.0
+- flux-core =0.59.0
+- versioneer =0.28
+- h5py
+- matplotlib =3.10.0
+- networkx =3.4.2
+- pygraphviz =1.14
+- pysqa =0.2.4
+- ipython =9.0.2
+- atomistics =0.2.4
+- qe =7.2
+- gpaw =24.6.0

.ci_support/environment-integration.yml

@@ -6,10 +6,10 @@ dependencies:
 - mpich
 - cloudpickle =3.1.1
 - mpi4py =4.0.1
-- pyzmq =26.2.1
-- h5py =3.12.1
+- pyzmq =26.3.0
+- h5py =3.13.0
 - networkx =3.4.2
 - pygraphviz =1.14
-- ipython =8.32.0
-- pysqa =0.2.3
+- ipython =9.0.2
+- pysqa =0.2.4
 - setuptools =75.1.0

.ci_support/environment-mpich.yml

@@ -6,10 +6,10 @@ dependencies:
 - openmpi
 - cloudpickle =3.1.1
 - mpi4py =4.0.1
-- pyzmq =26.2.1
-- h5py =3.12.1
+- pyzmq =26.3.0
+- h5py =3.13.0
 - networkx =3.4.2
 - pygraphviz =1.14
-- pysqa =0.2.3
-- ipython =8.32.0
+- pysqa =0.2.4
+- ipython =9.0.2
 - setuptools =75.1.0

.ci_support/environment-openmpi.yml

@@ -0,0 +1,6 @@
+channels:
+- conda-forge
+dependencies:
+- python
+- pylint
+- graphviz

.ci_support/environment-uml.yml

@@ -6,9 +6,9 @@ dependencies:
 - msmpi
 - cloudpickle =3.1.1
 - mpi4py =4.0.1
-- pyzmq =26.2.1
-- h5py =3.12.1
+- pyzmq =26.3.0
+- h5py =3.13.0
 - networkx =3.4.2
 - pygraphviz =1.14
-- ipython =8.32.0
+- ipython =9.0.2
 - setuptools =75.1.0

.ci_support/environment-win.yml

@@ -167,9 +167,38 @@ jobs:
       - name: Notebooks
         shell: bash -l {0}
         timeout-minutes: 5
-        run: >
-          flux start
-          .ci_support/build_notebooks.sh
+        run: |
+          papermill notebooks/1-single-node.ipynb notebooks/1-single-node-out.ipynb -k python3
+          flux start papermill notebooks/2-hpc-cluster.ipynb notebooks/2-hpc-cluster-out.ipynb -k python3
+          flux start papermill notebooks/3-hpc-job.ipynb notebooks/3-hpc-job-out.ipynb -k python3
+          papermill notebooks/4-developer.ipynb notebooks/4-developer-out.ipynb -k python3
+
+  notebooks_integration:
+    needs: [black]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Merge Notebook environment
+        run: |
+          echo -e "channels:\n  - conda-forge\n" > .condarc
+      - name: Setup Mambaforge
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          python-version: "3.12"
+          miniforge-version: latest
+          condarc-file: .condarc
+          environment-file: .ci_support/environment-integration.yml
+      - name: Install
+        shell: bash -l {0}
+        run: |
+          pip install versioneer[toml]==0.29
+          pip install . --no-deps --no-build-isolation
+      - name: Notebooks
+        shell: bash -l {0}
+        timeout-minutes: 20
+        run: |
+          flux start papermill notebooks/5-1-gpaw.ipynb notebooks/5-1-gpaw-out.ipynb -k python3
+          flux start papermill notebooks/5-2-quantum-espresso.ipynb notebooks/5-2-quantum-espresso-out.ipynb -k python3
 
   unittest_flux_mpich:
     needs: [black]
@@ -200,7 +229,7 @@ jobs:
       timeout-minutes: 5
       run: >
         flux start
-        python -m unittest tests/test_flux_executor.py tests/test_executor_backend_flux.py tests/test_cache_executor_pysqa_flux.py tests/test_plot_dependency_flux.py;
+        python -m unittest tests/test_fluxpythonspawner.py tests/test_fluxjobexecutor_plot.py tests/test_fluxjobexecutor.py tests/test_fluxclusterexecutor.py;
 
   unittest_flux_openmpi:
     needs: [black]
@@ -231,7 +260,7 @@ jobs:
       timeout-minutes: 5
       run: >
         flux start
-        coverage run -a --omit="executorlib/_version.py,tests/*" -m unittest tests/test_flux_executor.py tests/test_executor_backend_flux.py tests/test_cache_executor_pysqa_flux.py tests/test_plot_dependency_flux.py;
+        coverage run -a --omit="executorlib/_version.py,tests/*" -m unittest tests/test_fluxpythonspawner.py tests/test_fluxjobexecutor_plot.py tests/test_fluxjobexecutor.py tests/test_fluxclusterexecutor.py;
         coverage xml
       env:
         EXECUTORLIB_PMIX: "pmix"
@@ -259,9 +288,6 @@ jobs:
         - operating-system: ubuntu-latest
           python-version: '3.11'
 
-        - operating-system: ubuntu-latest
-          python-version: '3.10'
-
     steps:
     - uses: actions/checkout@v4
     - name: Conda config
@@ -300,9 +326,6 @@ jobs:
         - operating-system: ubuntu-latest
           python-version: '3.11'
 
-        - operating-system: ubuntu-latest
-          python-version: '3.10'
-
     steps:
     - uses: actions/checkout@v4
     - name: Conda config
@@ -385,3 +408,28 @@ jobs:
         env:
           PR_URL: ${{github.event.pull_request.html_url}}
           GH_TOKEN: ${{secrets.GITHUB_TOKEN}}
+
+  uml:
+    needs: [unittest_old, unittest_win, unittest_openmpi, unittest_mpich, unittest_flux_openmpi, unittest_flux_mpich, notebooks, benchmark, minimal, pip_check, mypy]
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Conda config
+      shell: bash -l {0}
+      run: echo -e "channels:\n  - conda-forge\n" > .condarc
+    - uses: conda-incubator/setup-miniconda@v3
+      with:
+        python-version: "3.13"
+        miniforge-version: latest
+        condarc-file: .condarc
+        environment-file: .ci_support/environment-uml.yml
+    - name: Test
+      shell: bash -l {0}
+      timeout-minutes: 10
+      run: |
+        pyreverse -o png -p ./${{ github.event.repository.name }} ./${{ github.event.repository.name }}
+        zip -m uml.zip *.png
+    - uses: actions/upload-artifact@v4
+      with:
+        name: uml.zip
+        path: uml.zip

.github/workflows/pipeline.yml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.9.6
+    rev: v0.11.2
     hooks:
       - id: ruff
         name: ruff lint

.pre-commit-config.yaml

@@ -1,7 +1,9 @@
 # executorlib
 [![Pipeline](https://github.com/pyiron/executorlib/actions/workflows/pipeline.yml/badge.svg)](https://github.com/pyiron/executorlib/actions/workflows/pipeline.yml)
 [![codecov](https://codecov.io/gh/pyiron/executorlib/graph/badge.svg?token=KFIO3R08H3)](https://codecov.io/gh/pyiron/executorlib)
-[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/pyiron/executorlib/HEAD?labpath=notebooks%2Fexamples.ipynb)
+[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/pyiron/executorlib/HEAD)
+[![status](https://joss.theoj.org/papers/7a4bced702e5ea10e4630bc4d5dbaad2/status.svg)](https://joss.theoj.org/papers/7a4bced702e5ea10e4630bc4d5dbaad2)
+![GitHub Repo stars](https://img.shields.io/github/stars/pyiron/executorlib)
 
 Up-scale python functions for high performance computing (HPC) with executorlib. 
 
@@ -139,16 +141,21 @@ as hierarchical job scheduler within the allocations.
   * [SLURM](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm)
   * [SLURM with Flux](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm-with-flux)
   * [Flux](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#flux)
+* [Application](https://executorlib.readthedocs.io/en/latest/application.html)
+  * [GPAW](https://executorlib.readthedocs.io/en/latest/5-1-gpaw.html)
+  * [Quantum Espresso](https://executorlib.readthedocs.io/en/latest/5-2-quantum-espresso.html)
 * [Trouble Shooting](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html)
   * [Filesystem Usage](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#filesystem-usage)
   * [Firewall Issues](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#firewall-issues)
   * [Message Passing Interface](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#message-passing-interface)
   * [Python Version](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#python-version)
   * [Resource Dictionary](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#resource-dictionary)
   * [SSH Connection](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#ssh-connection)
-* [Developer](https://executorlib.readthedocs.io/en/latest/4-developer.html)
-  * [Communication](https://executorlib.readthedocs.io/en/latest/4-developer.html#communication)
-  * [External Executables](https://executorlib.readthedocs.io/en/latest/4-developer.html#external-executables)
+* [Support & Contribution](https://executorlib.readthedocs.io/en/latest/4-developer.html)
+  * [Issues](https://executorlib.readthedocs.io/en/latest/4-developer.html#issues)
+  * [Pull Requests](https://executorlib.readthedocs.io/en/latest/4-developer.html#pull-requests)
   * [License](https://executorlib.readthedocs.io/en/latest/4-developer.html#license)
   * [Modules](https://executorlib.readthedocs.io/en/latest/4-developer.html#modules)
+  * [Communication](https://executorlib.readthedocs.io/en/latest/4-developer.html#communication)
+  * [External Executables](https://executorlib.readthedocs.io/en/latest/4-developer.html#external-executables)
 * [Interface](https://executorlib.readthedocs.io/en/latest/api.html)