Merge branch 'main' into fix_docu

Author: irr-github

.gitignore

@@ -14,15 +14,18 @@ __pycache__
 
 /site/**
 !/resources/data/shapefiles/
-!/resources/**/*.csv
+# !/resources/**/*.csv
 /resources/**/*
 /resources/derived_data/
 /resources/data/province_shapes
 /resources/data/landuse_availability/World_EEZ/*eez*
 /workflow/scripts/*.pdf
+/workflow/notebooks/*.py
+/workflow/notebooks/*.ipynb
 /results
 !/results/.gitkeep
 /cutouts
+/workflow/scripts/*.pdf
 /logs
 /benchmarks
 /tests/derived_data/**

Remind-PyPSA-coupling

@@ -0,0 +1 @@
+Subproject commit 25d56dc033957a964661f5f70c856886c8e7f530

config/default_config.yaml

@@ -4,7 +4,7 @@
 # DO NOT FORGET TO LOOK AT THE TECHNOLOGY CONFIGURATION FILES
 
 run:
-  name: unamed_run
+  name: 
 foresight: "overnight"
 
 paths:
@@ -35,6 +35,7 @@ scenario:
     - 2060
   heating_demand: ["positive"]
 
+# emission reduction scenarios (can be selected with scenario.co2_pathway)
 co2_scenarios: 
   exp175default:
     control: "reduction" # price | reduction | budget | None
@@ -47,15 +48,16 @@ co2_scenarios:
       '2045': 0.79092165
       '2050': 0.87480618
       '2055': 0.94359322
-      '2060': 1.0
+      '2060': 1
 
-# TODO Need to test timezones and functionality of end_year_plus1
+# Time dimension
+# TODO Need to test functionality of end_year_plus1
 snapshots:
   start: "01-01 00:00"
   end: "12-31 23:00"
   bounds: 'both'
-  freq: '5h'
-  # test weighings -> to be improved
+  freq: '5h' # down to 1h
+  # for weighings -> to be improved
   frequency: 5.
   # in case snapshots cross the planning horizon year
   end_year_plus1: false
@@ -88,6 +90,8 @@ atlite:
       dy: 5
       weather_year: 2020
 
+# renewable potentials and availabilities from Atlite. 
+# !CARE cutout should match atlite.cutout_name above
 renewable:
   onwind:
     cutout: China-2020c
@@ -151,7 +155,9 @@ reporting:
   # This ensures consistent capacity reporting across the network
   adjust_link_capacities_by_efficiency: true
 
-bus_suffix: [""," central heat"," decentral heat"," gas"," coal"]
+
+# Network carriers and technologies
+bus_suffix: [""," central heat"," decentral heat"," gas"," coal"] #TODO check used
 bus_carrier: {
     "": "AC",
     " central heat": "heat",
@@ -161,12 +167,15 @@ bus_carrier: {
 }
 
 Techs:
-  ## for add carriers and generators - but ugly to have nuclear and biomass and heatpupm in vre
+  ## tech selection
+  #TODO: rework vre_techs to split with clean_techs. Merge vre with non_dispatchable
   vre_techs: [ "onwind","offwind","solar","solar thermal","hydroelectricity", "nuclear","biomass","beccs","heat pump","resistive heater","Sabatier","H2 CHP", "fuel cell"]
+  non_dispatchable: ['Offshore Wind', 'Onshore Wind', 'Solar', 'Solar Residential', "Solar Thermal"]
   conv_techs: ["OCGT", "CCGT", "CHP gas", "gas boiler","coal boiler","coal power plant","CHP coal"]
   store_techs: ["H2","battery","water tanks","PHS"]
-  coal_cc: true # currently myopic pathway only.  CC = co2 cap 
-  hydrogen_lines: true # currently myopic pathway only
+  non_dispatchable: ["onwind", "offwind", "solar", "solar thermal"]
+  hydrogen_lines: true
+  coal_ccs_retrofit: true # currently myopic pathway only.  CC = co2 cap 
 
 ## add components (overwrites vre tech choice)
 heat_coupling: false
@@ -205,6 +214,7 @@ solving:
     clip_p_max_pu: 0.01
     skip_iterations: false
     track_iterations: false
+    export_duals: true   # Single switch for dual functionality
 
   solver:
     name: gurobi
@@ -276,6 +286,15 @@ solving:
 
   mem: 80000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2
 
+# brownfield options
+existing_capacities:
+  add: True # whether to add brownfield capacities
+  grouping_years: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019, 2020, 2025, 2030, 2035, 2040, 2045, 2050, 2055, 2060]
+  collapse_years: False # Treat as a single unit when preparing & solving network 
+  threshold_capacity: 1 # TODO UNIT
+  techs: ['coal','CHP coal', 'CHP gas', 'OCGT', 'CCGT', 'solar', 'onwind', 'offwind', 'nuclear']
+
+# transmission options
 lines:
   line_length_factor: 1.25
   expansion:
@@ -285,17 +304,44 @@ lines:
 security:
   line_margin: 70 # max percent of line capacity
 
-existing_capacities:
-  add: True 
-  grouping_years: [1980,1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019, 2020, 2025, 2030, 2035, 2040, 2045, 2050, 2055, 2060]
-  threshold_capacity: 1 # TODO UNIT
-  techs: ['coal','CHP coal', 'CHP gas', 'OCGT', 'CCGT', 'solar', 'solar thermal', 'onwind', 'offwind','coal boiler','ground heat pump','nuclear']
-
-fetch_regions:
-  simplify_tol: 0.5
+# contingencies (slower!) -> epsilon is fraction of vre gen or load for which reserves should exist
+operational_reserve:
+  activate: false
+  epsilon_load: 0.02
+  epsilon_vres: 0.02
+  contingency: 300000 # MW. Additional reserves on top of fractions
 
+# TODO integrate in workflow: split_provinces is currently not supported by build_loads and reporting
+nodes: 
+  split_provinces: False # split (some) provinces (admin l1) using admin l2  (counties/prefectures)
+  # exclude currently only applies if split_provinces is True, will be fixed with integrartion to workflow
+  exclude_provinces: ["Macau", "HongKong"] # list of provinces to exclude from splitting
+  splits:
+    # adminl2 groups per adminl1. {adminl1: {custom_name: [adminl2_names],..}}
+    InnerMongolia:
+      West: 
+        - "Hulunbuir"
+        - "Xing'an"
+        - "Tongliao"
+        - "Chifeng"
+        - "Xilin Gol"
+      East: 
+        - "Alxa"
+        - "Baotou"
+        - "Baynnur"
+        - "Hohhot"
+        - "Ordos"
+        - "Ulaan Chab"
+        - "Wuhai"
 
+# compression of netcdf network objects
 io:
   nc_compression:
     level: 4
-    zlib: True
+    zlib: True
+
+# simplification of shapes to save memory. If using GPS for plant node assignment a lower value is preferable.
+fetch_regions:  
+  simplify_tol: 
+    eez: 0.5
+    land: 0.05

config/pik_hpc_profile/config.yaml

@@ -46,8 +46,8 @@ cluster-generic-cancel-cmd: scancel
 
 # now define the resources
 default-resources:
-  partition: standard
-  qos: short
+  partition: priority
+  qos: priority
   mem_mb: 2000
   time : 10
   threads: 1
@@ -57,7 +57,7 @@ set-resources:
     mem_mb: 80000
     time: 80
   solve_networks:
-    mem_mb: 60000
+    mem_mb: 90000
     time : 80
     threads: 6
   build_cutout:
@@ -115,9 +115,10 @@ set-resources:
 # GROUPS
 
 groups: 
-  plot_network: plot
-  plot_statistics: plot
-  plot_snapshots: plot
+  # plot_network: plot
+  # plot_statistics: plot
+  # plot_snapshots: plot
+  prep_capacities: prepare_baseyear_capacities
   build_load_profiles: build_load
   prepare_networks: prep_network
   add_existing_baseyear: prep_network2
@@ -127,4 +128,5 @@ group-components:
   build_load: 3
   prep_network: 3
   prep_network2: 3
-  summary: 3
+  summary: 3
+  prep_capacities: 3

config/plot_config.yaml

@@ -58,7 +58,8 @@ plotting:
   costs_threshold: 2.e+6 # Eur
   energy_threshold: 5.e+6 # MWh
   capacity_threshold: 1.e+4 # MW
-
+  market_value:
+      min_gen_share: 0.01
   loads: ['electricity', 'decentral heat', 'central heat']
   vre_techs: [ "onwind", "offwind", "solar"]
   conv_techs: [ "gas", "gas-heat", "gas-AC", "AC-gas", "coal", "coal-AC","solar thermal","AC-heat","AC-AC","stations-AC","uranium", "H2-AC", "H2-gas",]
@@ -139,12 +140,15 @@ plotting:
     Gas: '#e0986c'
     OCGT: '#e0986c'
     OCGT gas: '#e0986c'
+    gas ocgt: '#e0986c'
     gas OCGT: '#e0986c'
+    Open-Cycle Gas: '#e0986c'
     OCGT marginal: '#e0986c'
     OCGT-heat: '#e0986c'
     CCGT: '#a85522'
-    CCGT gas: '#a85522'
+    CCGT marginal: '#a85522'
     gas CCGT: '#a85522'
+    CCGT gas: '#a85522'
     gas ccs: '#a85522'
     CCGT-CCS: '#76855B'
     gas CCGT-CCS: '#76855B'
@@ -161,9 +165,6 @@ plotting:
     fossil gas: '#e05b09'
     natural gas: '#e05b09'
     biogas to gas: '#e36311'
-    CCGT: '#a85522'
-    CCGT marginal: '#a85522'
-    gas CCGT: '#a85522'
     allam: '#B98F76'
     gas for industry co2 to atmosphere: '#692e0a'
     gas for industry co2 to stored: '#8a3400'
@@ -172,8 +173,6 @@ plotting:
     gas pipeline: '#ebbca0'
     gas pipeline new: '#a87c62'
     gas-AC: "grey"
-
-
     # hot water storage
     water tanks: '#e69487'
     residential rural water tanks: '#f7b7a3'
@@ -226,8 +225,9 @@ plotting:
     CHP coal: '#8a5751'
     CHP coal generator: '#8a5751'
     coal heat: '#8a7f7d'
-    urban central gas CHP: '#8f6e6a'
+    gas CHP: '#8f6e6a'
     CHP gas: '#8f6e6a'
+    central gas CHP: '#8f6e6a'
     CHP gas generator: '#8f6e6a'
     CHP gas boiler: '#8f6e6a'
     CHP CC: '#634643'
@@ -401,6 +401,7 @@ plotting:
     CHP: "Combined Heat and Power"
     OCGT: "Open-Cycle Gas"
     gas OCGT: "Open-Cycle Gas"
+    Gas Ocgt: "Open-Cycle Gas"
     CCGT: "Combined-Cycle Gas"
     gas CCGT: "Combined-Cycle Gas"
     gas ccs: "Gas with Carbon Capture"
@@ -411,6 +412,7 @@ plotting:
     # generatord
     battery: "Battery Storage"
     battery discharger: "Battery Discharger"
+    # coal: "Coal"
     coal: "Coal Power Plant"
     coal cc: "Coal with carbon capture"
     coal ccs: "Coal with carbon capture"
@@ -419,15 +421,10 @@ plotting:
     hydro: "Reservoir & Dam"
     H2 fuel cell: "H2 Fuel Cell"
     PHS: "Pumped Hydro Storage"
-    offwind: "Offshore Wind"
-    onwind: "Onshore Wind"
-    solar: "Solar"
     # H2: "Hydrogen Power"
     ror: "Run of River"
-    solar thermal: "Solar Thermal"
     uranium: "Nuclear"
     hydroelectricity: "Hydroelectricity"
-    coal: "Coal"
     offwind: "Offshore Wind"
     onwind: "Onshore Wind"
     solar: "Solar"

workflow/Snakefile

@@ -436,33 +436,20 @@ rule dag:
         """
 
 onerror:
-    log_out = expand(LOG_DIR + "/_latest_snakemake.log", **config["scenario"])[0]
     res_dir = expand(RESULTS_DIR, **config["scenario"])[0]
     if not os.path.exists(res_dir):
         os.makedirs(res_dir)
-    if not os.path.exists(LOG_DIR):
-        os.makedirs(LOG_DIR)
-    # log path is available from snakemake as "log"
     try:
-        shutil.copy(log, os.path.dirname(log_out))
         yaml.dump(config, open(res_dir + "/run_config.yaml", "w"))
     except FileNotFoundError as e:
         logging.warning(f"Could not copy log file: {e}")
     except TypeError as e:
         logging.warning(f"Could not copy config file: {e}")
-        logging.warning(f"Copying log file to {str(log_out)} failed?")
-        logging.warning(f"saving config file to {str(res_dir)}/run_config.yaml failed?")
 
 onsuccess:
-    log_out = expand(LOG_DIR + "/_latest_snakemake.log", **config["scenario"])[0]
     res_dir = expand(RESULTS_DIR, **config["scenario"])[0]
     try:
-        shutil.copy(log, log_out)
-        yaml.dump(config, open(res_dir + "/run_config.yaml", "w"))
-    except FileNotFoundError as e:
-        logging.warning(f"Could not copy log file: {e}")
+        with open(res_dir + "/run_config.yaml", "w") as f:
+            yaml.dump(config, f )
     except TypeError as e:
         logging.warning(f"Could not copy config file: {e}")
-        logging.warning(f"Copying log file to {str(log_out)} (type {type(log_out)}) failed?")
-        logging.info(f"source was {str(log)}")
-        logging.warning(f"sav config file to {str(res_dir)}/run_config.yaml failed?")

workflow/envs/environment.yaml

@@ -10,8 +10,8 @@ dependencies:
   - python>=3.12
   - pip
 
-  - pypsa=0.31
-  - atlite>=0.2.6
+  - pypsa=0.35.1
+  - atlite>=0.4.0
   - dask
   - snakemake-storage-plugin-http
   # - pyasyncore
@@ -23,7 +23,7 @@ dependencies:
   - openpyxl
   - pycountry
   - seaborn
-  - snakemake-minimal>=8.0
+  - snakemake-minimal>=9.0
   - memory_profiler
   - yaml
   - pytables