Merge pull request pik-piam#68 from beijingzyl/testupdate

Test update

Author: beijingzyl

.github/workflows/run_tests.yml

@@ -15,28 +15,42 @@ jobs:
     runs-on: ubuntu-latest  # Set the environment to Ubuntu
 
     steps:
-    # Step 1: Checkout the repository code
+    # Step 1: Checkout the repository
     - uses: actions/checkout@v4
-    # Step 2: Set up Conda
-    - name: Set up Conda
+
+    # Step 2: Install Miniforge (no env creation here)
+    - name: Setup Miniforge
       uses: conda-incubator/setup-miniconda@v3
       with:
-        miniconda-version: 'latest'  # Use the latest version of Miniconda
-        environment-file: workflow/envs/environment.yaml  # Point to your conda environment file (if exists)
+        miniforge-version: latest
+        installation-dir: /home/runner/miniforge3
+        use-mamba: true
         auto-activate-base: false
-        activate-environment: pypsa-china
-    # /3 cache the Conda environment (for faster CI)
-    - name: Cache Conda environment
-      uses: actions/cache@v4  
+        channels: conda-forge,bioconda
+        channel-priority: strict
+        pkgs-dirs: /home/runner/.conda/pkgs
+
+    # Step 3: Restore full Conda environment cache AFTER setup
+    - name: Restore Conda env cache
+      id: env-cache
+      uses: actions/cache@v4
       with:
-        path: ~/miniconda3
-        key: ${{ runner.os }}-conda-${{ hashFiles('workflow/envs/environment.yaml') }}
-        restore-keys: |
-          ${{ runner.os }}-conda-
-    # 4/ Run tests using pytest
-    - name: Run pytest
+        path: |
+          /home/runner/miniforge3/envs/pypsa-china
+          /home/runner/.conda/pkgs
+          /home/runner/.cache/mamba
+        key: ${{ runner.os }}-conda-env-${{ hashFiles('workflow/envs/environment.yaml') }}
+
+    # Step 4: Create env only if cache miss
+    - name: Create env
+      if: steps.env-cache.outputs.cache-hit != 'true'
       shell: bash -el {0}
       run: |
-        conda install pytest
-        pytest -v -s  # Run pytest (make sure your tests are discovered and run)
+        conda config --add envs_dirs /home/runner/miniforge3/envs
+        mamba env create -n pypsa-china -f workflow/envs/environment.yaml
 
+    # Step 5: Run tests
+    - name: Run pytest
+      shell: bash -el {0}
+      run: |
+        conda run -n pypsa-china pytest -v -s

.pre-commit-config.yaml

@@ -1,14 +1,22 @@
 repos:
-  - repo: https://github.com/psf/black
-    rev: 22.3.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.12.10
     hooks:
-      - id: black
-        args: [--line-length=100]
-        language_version: python3.12
+      # Run the linter.
+      - id: ruff-check
+        args: [ --fix ]
+      # Run the formatter.
+      - id: ruff-format
+      
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
     hooks:
       - id: check-yaml
       - id: check-case-conflict
       - id: end-of-file-fixer
       - id: trailing-whitespace
+  - repo: https://github.com/kynan/nbstripout
+    rev: 0.8.1
+    hooks:
+      - id: nbstripout

.ruff.toml

@@ -0,0 +1,55 @@
+line-length = 100
+target-version = "py312"
+
+# Exclude notebook files from linting
+exclude = [
+    "**/*.ipynb",
+    "workflow/notebooks/**",
+]
+
+[lint]
+select = [
+    "F",   # pyflakes
+    "E",   # pycodestyle: Error
+    "W",   # pycodestyle: Warning
+    "I",   # isort
+    "D",   # pydocstyle
+    "UP",  # pyupgrade
+    "TID", # flake8-tidy-imports
+]
+ignore = [
+    "ANN401", # Dynamically typed expressions are forbidden
+    "E712", # comparison to False should be 'if cond is False:' or 'if not cond:'
+    "E741", # ambiguous variable names
+    "D203", # 1 blank line required before class docstring
+    "D212", # Multi-line docstring summary should start at the second line
+    "D401", # First line should be in imperative mood
+]
+
+[lint.per-file-ignores]
+# Ignore docstring and line length issues in all Python files
+
+"**/*.py" = [
+    'E501', # line too long
+    'D107', # Missing docstring in __init__
+    'D200', # One-line docstring should fit on one line with quotes
+    'D202', # No blank lines allowed after function docstring
+    'D205', # 1 blank line required between summary line and description
+    'D400', # First line should end with a period
+    'D404', # First word of the docstring should not be "This"
+    'D413', # Missing blank line after last section
+    'D415', # First line should end with a period, question mark, or exclamation point
+    'D417', # Missing argument descriptions in the docstring
+    "W291", # trailing whitespace
+]
+
+[format]
+quote-style = "double"
+indent-style = "space"
+
+[lint.pydocstyle]
+convention = "google"
+
+[lint.isort]
+combine-as-imports = true
+known-first-party = ["workflow"]

Remind-PyPSA-coupling

@@ -2,7 +2,6 @@
 # https://github.com/LUMC/slurm-cluster-status/blob/master/README.md
 import argparse
 import subprocess
-import time
 
 STATE_MAP = {
     "BOOT_FAIL": "failed",
@@ -20,25 +19,24 @@
     "RESIZING": "running",
     "SUSPENDED": "running",
     "TIMEOUT": "failed",
-    "UNKNOWN": "running"
+    "UNKNOWN": "running",
 }
 
 
 def fetch_status(batch_id):
-    """fetch the status for the batch id"""
-    sacct_args = ["sacct", "-j",  batch_id, "-o", "State", "--parsable2",
-                  "--noheader"]
+    """Fetch the status for the batch id"""
+    sacct_args = ["sacct", "-j", batch_id, "-o", "State", "--parsable2", "--noheader"]
 
     try:
         output = subprocess.check_output(sacct_args).decode("utf-8").strip()
     except Exception:
         # If sacct fails for whatever reason, assume its temporary and return 'running'
-        output = 'UNKNOWN'
+        output = "UNKNOWN"
 
     # Sometimes, sacct returns nothing, in which case we assume it is temporary
     # and return 'running'
     if not output:
-        output = 'UNKNOWN'
+        output = "UNKNOWN"
 
     # The first output is the state of the overall job
     # See
@@ -55,8 +53,9 @@ def fetch_status(batch_id):
     try:
         return STATE_MAP[job_status]
     except KeyError:
-        raise NotImplementedError(f"Encountered unknown status '{job_status}' "
-                                  f"when parsing output:\n'{output}'")
+        raise NotImplementedError(
+            f"Encountered unknown status '{job_status}' when parsing output:\n'{output}'"
+        )
 
 
 if __name__ == "__main__":
@@ -65,4 +64,4 @@ def fetch_status(batch_id):
     args = parser.parse_args()
 
     status = fetch_status(args.batch_id)
-    print(status)
+    print(status)

config/pik_hpc_profile/status-check.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+
 import mkdocs_gen_files
 
 # Set the root and scripts directory

docs/autogen_ref_pages.py

@@ -4,19 +4,15 @@
 """
 
 import pathlib
-import yaml
-import pytest
+from hashlib import sha256 as hash256
 from os import PathLike
-import matplotlib
-import shutil
 
-from hashlib import sha256 as hash256
-from os import remove
+import matplotlib
+import pytest
+import yaml
 
 # from filelock import BaseFileLock
-
-from constants import TESTS_RUNNAME, TESTS_CUTOUT
-from _helpers import PathManager
+from constants import TESTS_CUTOUT, TESTS_RUNNAME
 
 DEFAULT_CONFIG = pathlib.Path(pathlib.Path.cwd(), "config", "default_config.yaml")
 TECH_CONFIG = pathlib.Path(pathlib.Path.cwd(), "config", "technology_config.yaml")
@@ -29,7 +25,7 @@ def set_matplotlib_backend():
 
 
 def load_config(config_path: PathLike) -> dict:
-    """load a config file
+    """Load a config file
     Args:
         config_path (PathLike): the path to the config file
 
@@ -43,17 +39,21 @@ def load_config(config_path: PathLike) -> dict:
 
 @pytest.fixture(scope="module")
 def make_snakemake_test_config(tmp_path_factory) -> dict:
-    """make a test config for snamekemake based on the default config
+    """Make a test config for snamekemake based on the default config
     Example:
         conf_dict = make_snamkemake_test_config({"scenario":{"planning_horizons":2030}})
+
     Returns:
         dict: the test config
     """
 
     def make(
-        time_res=24, plan_year=2040, start_d="01-02 00:00", end_d="01-02 23:00", **kwargs
+        time_res=24,
+        plan_year=2040,
+        start_d="01-02 00:00",
+        end_d="01-02 23:00",
+        **kwargs,
     ) -> dict:
-
         base_config = load_config(DEFAULT_CONFIG)
         # base_config.update(load_config(TECH_CONFIG))
         base_config.update(kwargs)
@@ -76,7 +76,6 @@ def make(
         test_config["run"]["name"] = TESTS_RUNNAME
         test_config["run"]["is_test"] = True
         test_config["paths"]["costs_dir"] = None
-        
 
         # mock the atlite cutout config
         test_config["atlite"]["cutout_name"] = TESTS_CUTOUT
@@ -94,7 +93,6 @@ def make_test_config_file(make_snakemake_test_config, tmpdir_factory, request):
     """Fixture to save a temp config file for testing, return its path,
     and clean up after module.
     """
-
     # Get parameters passed via pytest.mark.parametrize
     time_res = request.param.get("time_res", 24)
     plan_year = request.param.get("plan_year", 2040)