Merge remote-tracking branch 'upstream/master' into multiyear

.git-blame-ignore-revs

@@ -6,3 +6,4 @@
 5d1ef8a64055a039aa4a0834d2d26fe7752fe9a0
 92080b1cd2ca5f123158571481722767b99c2b27
 13769f90af4500948b0376d57df4cceaa13e78b5
+9865a970893d9e515786f33c629b14f71645bf1e

.gitattributes

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 

.github/workflows/ci.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2021-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 
@@ -32,7 +32,14 @@ jobs:
         - ubuntu-latest
         - macos-latest
         - windows-latest
-
+        inhouse:
+        - stable
+        - master
+        exclude:
+        - os: macos-latest
+          inhouse: master
+        - os: windows-latest
+          inhouse: master
     runs-on: ${{ matrix.os }}
 
     defaults:
@@ -46,16 +53,6 @@ jobs:
       run: |
         echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc
 
-    - name: Add solver to environment
-      run: |
-        echo -e "- glpk\n- ipopt<3.13.3" >> envs/environment.yaml
-      if: ${{ matrix.os }} == 'windows-latest'
-
-    - name: Add solver to environment
-      run: |
-        echo -e "- glpk\n- ipopt" >> envs/environment.yaml
-      if: ${{ matrix.os }} != 'windows-latest'
-
     - name: Setup micromamba
       uses: mamba-org/setup-micromamba@v1
       with:
@@ -66,6 +63,11 @@ jobs:
         cache-environment: true
         cache-downloads: true
 
+    - name: Install inhouse packages
+      run: |
+        pip install git+https://github.com/PyPSA/atlite.git@master git+https://github.com/PyPSA/powerplantmatching.git@master git+https://github.com/PyPSA/linopy.git@master
+      if: ${{ matrix.inhouse }} == 'master'
+
     - name: Set cache dates
       run: |
         echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV
@@ -79,14 +81,10 @@ jobs:
         key: data-cutouts-${{ env.WEEK }}-${{ env.DATA_CACHE_NUMBER }}
 
     - name: Test snakemake workflow
-      run: |
-        snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime
-        snakemake -call all --configfile config/test/config.overnight.yaml --rerun-triggers=mtime
-        snakemake -call all --configfile config/test/config.myopic.yaml --rerun-triggers=mtime
-        snakemake -call all --configfile config/test/config.perfect.yaml --rerun-triggers=mtime
+      run: ./test.sh
 
     - name: Upload artifacts
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4.3.0
       with:
         name: resources-results
         path: |
@@ -94,3 +92,4 @@ jobs:
           results
         if-no-files-found: warn
         retention-days: 1
+      if: matrix.os == 'ubuntu' && matrix.inhouse == 'stable'

.gitignore

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 
@@ -20,10 +20,18 @@ gurobi.log
 /notebooks
 /data
 /cutouts
-
+/tmp
 doc/_build
 
+/scripts/old
+/scripts/create_scenarios.py
+/config/create_scenarios.py
+
+config/config.yaml
+config/scenarios.yaml
+
 config.yaml
+config/config.yaml
 
 dconf
 /data/links_p_nom.csv
@@ -53,25 +61,15 @@ d1gam3xoknrgr2.cloudfront.net/
 *.nc
 
 *~
-/scripts/old
 
 *.pyc
-/cutouts
-/tmp
-/pypsa
 
 *.xlsx
 
-config.yaml
-
-doc/_build
-
 *.xls
 
 *.geojson
 
 *.ipynb
 
-data/costs_*
-
 merger-todos.md

.pre-commit-config.yaml

@@ -51,7 +51,7 @@ repos:
 
   # Formatting with "black" coding style
 - repo: https://github.com/psf/black-pre-commit-mirror
-  rev: 23.12.1
+  rev: 24.2.0
   hooks:
       # Format Python files
   - id: black
@@ -74,7 +74,7 @@ repos:
 
   # Format Snakemake rule / workflow files
 - repo: https://github.com/snakemake/snakefmt
-  rev: v0.8.5
+  rev: v0.10.0
   hooks:
   - id: snakefmt
 
@@ -87,6 +87,6 @@ repos:
 
   # Check for FSFE REUSE compliance (licensing)
 - repo: https://github.com/fsfe/reuse-tool
-  rev: v2.1.0
+  rev: v3.0.1
   hooks:
   - id: reuse

.readthedocs.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 

.reuse/dep5

@@ -4,33 +4,33 @@ Upstream-Contact: Tom Brown <t.brown@tu-berlin.de>
 Source: https://github.com/pypsa/pypsa-eur
 
 Files: doc/img/*
-Copyright: 2019-2023 The PyPSA-Eur Authors
+Copyright: 2019-2024 The PyPSA-Eur Authors
 License: CC-BY-4.0
 
 Files: doc/data.csv
-Copyright: 2019-2023 The PyPSA-Eur Authors
+Copyright: 2019-2024 The PyPSA-Eur Authors
 License: CC-BY-4.0
 
 Files: doc/configtables/*
-Copyright: 2019-2023 The PyPSA-Eur Authors
+Copyright: 2019-2024 The PyPSA-Eur Authors
 License: CC-BY-4.0
 
 Files: data/*
-Copyright: 2017-2023 The PyPSA-Eur Authors
+Copyright: 2017-2024 The PyPSA-Eur Authors
 License: CC-BY-4.0
 
 Files: .github/*
-Copyright: 2019-2023 The PyPSA-Eur Authors
+Copyright: 2019-2024 The PyPSA-Eur Authors
 License: CC0-1.0
 
 Files: matplotlibrc
-Copyright: 2017-2023 The PyPSA-Eur Authors
+Copyright: 2017-2024 The PyPSA-Eur Authors
 License: CC0-1.0
 
 Files: borg-it
-Copyright: 2017-2023 The PyPSA-Eur Authors
+Copyright: 2017-2024 The PyPSA-Eur Authors
 License: CC0-1.0
 
 Files: graphics/*
-Copyright: 2017-2023 The PyPSA-Eur Authors
+Copyright: 2017-2024 The PyPSA-Eur Authors
 License: CC-BY-4.0

.sync-send

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2021-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 

CITATION.cff

@@ -6,7 +6,7 @@ cff-version: 1.1.0
 message: "If you use this package, please cite it in the following way."
 title: "PyPSA-Eur: An open sector-coupled optimisation model of the European energy system"
 repository: https://github.com/pypsa/pypsa-eur
-version: 0.8.1
+version: 0.10.0
 license: MIT
 authors:
   - family-names: Brown

LICENSES/MIT.txt

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright 2017-2023 The PyPSA-Eur Authors
+Copyright 2017-2024 The PyPSA-Eur Authors
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

README.md

@@ -1,5 +1,5 @@
 <!--
-SPDX-FileCopyrightText: 2017-2023 The PyPSA-Eur Authors
+SPDX-FileCopyrightText: 2017-2024 The PyPSA-Eur Authors
 SPDX-License-Identifier: CC-BY-4.0
 -->
 

Snakefile

@@ -1,36 +1,53 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 
 from os.path import normpath, exists
 from shutil import copyfile, move, rmtree
+from pathlib import Path
+import yaml
 
 from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider
-
-HTTP = HTTPRemoteProvider()
-
 from snakemake.utils import min_version
 
+from scripts._helpers import path_provider
+
 min_version("7.7")
+HTTP = HTTPRemoteProvider()
+
+default_files = {
+    "config/config.default.yaml": "config/config.yaml",
+    "config/scenarios.template.yaml": "config/scenarios.yaml",
+}
+for template, target in default_files.items():
+    target = os.path.join(workflow.current_basedir, target)
+    template = os.path.join(workflow.current_basedir, template)
+    if not exists(target) and exists(template):
+        copyfile(template, target)
 
 
-if not exists("config/config.yaml"):
-    copyfile("config/config.default.yaml", "config/config.yaml")
-
-
+configfile: "config/config.default.yaml"
 configfile: "config/config.yaml"
 
 
-COSTS = f"data/costs_{config['costs']['year']}.csv"
-ATLITE_NPROCESSES = config["atlite"].get("nprocesses", 4)
+run = config["run"]
+scenarios = run.get("scenarios", {})
+if run["name"] and scenarios.get("enable"):
+    fn = Path(scenarios["file"])
+    scenarios = yaml.safe_load(fn.read_text())
+    RDIR = "{run}/"
+    if run["name"] == "all":
+        config["run"]["name"] = list(scenarios.keys())
+elif run["name"]:
+    RDIR = run["name"] + "/"
+else:
+    RDIR = ""
 
-run = config.get("run", {})
-RDIR = run["name"] + "/" if run.get("name") else ""
-CDIR = RDIR if not run.get("shared_cutouts") else ""
+logs = path_provider("logs/", RDIR, run["shared_resources"])
+benchmarks = path_provider("benchmarks/", RDIR, run["shared_resources"])
+resources = path_provider("resources/", RDIR, run["shared_resources"])
 
-LOGS = "logs/" + RDIR
-BENCHMARKS = "benchmarks/" + RDIR
-RESOURCES = "resources/" + RDIR if not run.get("shared_resources") else "resources/"
+CDIR = "" if run["shared_cutouts"] else RDIR
 RESULTS = "results/" + RDIR
 
 
@@ -74,10 +91,19 @@ if config["foresight"] == "perfect":
 
 rule all:
     input:
-        RESULTS + "graphs/costs.pdf",
+        expand(RESULTS + "graphs/costs.pdf", run=config["run"]["name"]),
     default_target: True
 
 
+rule create_scenarios:
+    output:
+        config["run"]["scenarios"]["file"],
+    conda:
+        "envs/retrieve.yaml"
+    script:
+        "config/create_scenarios.py"
+
+
 rule purge:
     run:
         import builtins
@@ -98,9 +124,9 @@ rule dag:
     message:
         "Creating DAG of workflow."
     output:
-        dot=RESOURCES + "dag.dot",
-        pdf=RESOURCES + "dag.pdf",
-        png=RESOURCES + "dag.png",
+        dot=resources("dag.dot"),
+        pdf=resources("dag.pdf"),
+        png=resources("dag.png"),
     conda:
         "envs/environment.yaml"
     shell:
@@ -126,6 +152,7 @@ rule sync:
     shell:
         """
         rsync -uvarh --ignore-missing-args --files-from=.sync-send . {params.cluster}
+        rsync -uvarh --no-g {params.cluster}/resources . || echo "No resources directory, skipping rsync"
         rsync -uvarh --no-g {params.cluster}/results . || echo "No results directory, skipping rsync"
         rsync -uvarh --no-g {params.cluster}/logs . || echo "No logs directory, skipping rsync"
         """

config/config.default.yaml

@@ -1,9 +1,9 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#top-level-configuration
-version: 0.8.1
+version: 0.10.0
 tutorial: false
 
 logging:
@@ -21,6 +21,9 @@ remote:
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#run
 run:
   name: ""
+  scenarios:
+    enable: false
+    file: config/scenarios.yaml
   disable_progressbar: false
   shared_resources: false
   shared_cutouts: true
@@ -45,7 +48,7 @@ scenario:
   opts:
   - ''
   sector_opts:
-  - Co2L0-3H-T-H-B-I-A-solar+p3-dist1
+  - Co2L0-3H-T-H-B-I-A-dist1
   planning_horizons:
   # - 2020
   # - 2030
@@ -76,7 +79,8 @@ enable:
   build_natura_raster: false
   retrieve_natura_raster: true
   custom_busmap: false
-  drop_leap_days: true
+  drop_leap_day: true
+
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#co2-budget
 co2_budget:
@@ -91,7 +95,9 @@ co2_budget:
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#electricity
 electricity:
   voltages: [220., 300., 380., 500., 750.]
+  gaslimit_enable: false
   gaslimit: false
+  co2limit_enable: false
   co2limit: 7.75e+7
   co2base: 1.487e+9
   agg_p_nom_limits: data/agg_p_nom_minmax.csv
@@ -112,8 +118,9 @@ electricity:
     Store: [battery, H2]
     Link: [] # H2 pipeline
 
-  powerplants_filter: (DateOut >= 2022 or DateOut != DateOut)
+  powerplants_filter: (DateOut >= 2023 or DateOut != DateOut) and not (Country == 'Germany' and Fueltype == 'Nuclear')
   custom_powerplants: false
+  everywhere_powerplants: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass]
 
   conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass]
   renewable_carriers: [solar, onwind, offwind-ac, offwind-dc, hydro]
@@ -128,6 +135,10 @@ electricity:
       Onshore: [onwind]
       PV: [solar]
 
+  autarky:
+    enable: false
+    by_country: false
+
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#atlite
 atlite:
   default_cutout: europe-2013-era5
@@ -169,6 +180,10 @@ renewable:
       grid_codes: [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32]
       distance: 1000
       distance_grid_codes: [1, 2, 3, 4, 5, 6]
+    luisa: false
+      # grid_codes: [1111, 1121, 1122, 1123, 1130, 1210, 1221, 1222, 1230, 1241, 1242]
+      # distance: 1000
+      # distance_grid_codes: [1111, 1121, 1122, 1123, 1130, 1210, 1221, 1222, 1230, 1241, 1242]
     natura: true
     excluder_resolution: 100
     clip_p_max_pu: 1.e-2
@@ -181,6 +196,7 @@ renewable:
     capacity_per_sqkm: 2
     correction_factor: 0.8855
     corine: [44, 255]
+    luisa: false # [0, 5230]
     natura: true
     ship_threshold: 400
     max_depth: 50
@@ -196,6 +212,7 @@ renewable:
     capacity_per_sqkm: 2
     correction_factor: 0.8855
     corine: [44, 255]
+    luisa: false # [0, 5230]
     natura: true
     ship_threshold: 400
     max_depth: 50
@@ -210,9 +227,10 @@ renewable:
       orientation:
         slope: 35.
         azimuth: 180.
-    capacity_per_sqkm: 1.7
+    capacity_per_sqkm: 5.1
     # correction_factor: 0.854337
     corine: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 26, 31, 32]
+    luisa: false # [1111, 1121, 1122, 1123, 1130, 1210, 1221, 1222, 1230, 1241, 1242, 1310, 1320, 1330, 1410, 1421, 1422, 2110, 2120, 2130, 2210, 2220, 2230, 2310, 2410, 2420, 3210, 3320, 3330]
     natura: true
     excluder_resolution: 100
     clip_p_max_pu: 1.e-2
@@ -245,7 +263,7 @@ lines:
     750.: "Al/St 560/50 4-bundle 750.0"
   s_max_pu: 0.7
   s_nom_max: .inf
-  max_extension: .inf
+  max_extension: 20000 #MW
   length_factor: 1.25
   reconnect_crimea: true
   under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity
@@ -260,7 +278,7 @@ lines:
 links:
   p_max_pu: 1.0
   p_nom_max: .inf
-  max_extension: .inf
+  max_extension: 30000 #MW
   include_tyndp: true
   under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity
 
@@ -270,7 +288,7 @@ transformers:
   s_nom: 2000.
   type: ''
 
-# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#load
+# docs-load in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#load
 load:
   power_statistics: true
   interpolate_limit: 3
@@ -296,6 +314,7 @@ pypsa_eur:
   - offwind-dc
   - solar
   - ror
+  - nuclear
   StorageUnit:
   - PHS
   - hydro
@@ -303,7 +322,7 @@ pypsa_eur:
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#energy
 energy:
-  energy_totals_year: 2011
+  energy_totals_year: 2013
   base_emissions_year: 1990
   eurostat_report_year: 2016
   emissions: CO2
@@ -346,6 +365,7 @@ existing_capacities:
   grouping_years_power: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030]
   grouping_years_heat: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019] # these should not extend 2020
   threshold_capacity: 10
+  default_heating_lifetime: 20
   conventional_carriers:
   - lignite
   - coal
@@ -354,15 +374,23 @@ existing_capacities:
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#sector
 sector:
+  transport: true
+  heating: true
+  biomass: true
+  industry: true
+  agriculture: true
   district_heating:
     potential: 0.6
     progress:
       2020: 0.0
+      2025: 0.15
       2030: 0.3
+      2035: 0.45
       2040: 0.6
+      2045: 0.8
       2050: 1.0
     district_heating_loss: 0.15
-  cluster_heat_buses: false
+  cluster_heat_buses: true
   bev_dsm_restriction_value: 0.75
   bev_dsm_restriction_time: 7
   transport_heating_deadband_upper: 20.
@@ -382,18 +410,27 @@ sector:
   v2g: true
   land_transport_fuel_cell_share:
     2020: 0
-    2030: 0.05
-    2040: 0.1
-    2050: 0.15
+    2025: 0
+    2030: 0
+    2035: 0
+    2040: 0
+    2045: 0
+    2050: 0
   land_transport_electric_share:
     2020: 0
-    2030: 0.25
-    2040: 0.6
-    2050: 0.85
+    2025: 0.15
+    2030: 0.3
+    2035: 0.45
+    2040: 0.7
+    2045: 0.85
+    2050: 1
   land_transport_ice_share:
     2020: 1
+    2025: 0.85
     2030: 0.7
+    2035: 0.55
     2040: 0.3
+    2045: 0.15
     2050: 0
   transport_fuel_cell_efficiency: 0.5
   transport_internal_combustion_efficiency: 0.3
@@ -407,18 +444,27 @@ sector:
   shipping_hydrogen_liquefaction: false
   shipping_hydrogen_share:
     2020: 0
+    2025: 0
     2030: 0
+    2035: 0
     2040: 0
+    2045: 0
     2050: 0
   shipping_methanol_share:
     2020: 0
+    2025: 0.15
     2030: 0.3
+    2035: 0.5
     2040: 0.7
+    2045: 0.85
     2050: 1
   shipping_oil_share:
     2020: 1
+    2025: 0.85
     2030: 0.7
+    2035: 0.5
     2040: 0.3
+    2045: 0.15
     2050: 0
   shipping_methanol_efficiency: 0.46
   shipping_oil_efficiency: 0.40
@@ -447,8 +493,10 @@ sector:
     decentral: 3
     central: 180
   boilers: true
+  resistive_heaters: true
   oil_boilers: false
   biomass_boiler: true
+  overdimension_individual_heating: 1.1  #to cover demand peaks bigger than data
   chp: true
   micro_chp: false
   solar_thermal: true
@@ -464,6 +512,9 @@ sector:
   hydrogen_turbine: false
   SMR: true
   SMR_cc: true
+  regional_methanol_demand: false
+  regional_oil_demand: false
+  regional_coal_demand: false
   regional_co2_sequestration_potential:
     enable: false
     attribute: 'conservative estimate Mt'
@@ -476,6 +527,7 @@ sector:
   co2_sequestration_lifetime: 50
   co2_spatial: false
   co2network: false
+  co2_network_cost_factor: 1
   cc_fraction: 0.9
   hydrogen_underground_storage: true
   hydrogen_underground_storage_locations:
@@ -492,9 +544,20 @@ sector:
   use_methanation_waste_heat: true
   use_fuel_cell_waste_heat: true
   use_electrolysis_waste_heat: true
+  electricity_transmission_grid: true
   electricity_distribution_grid: true
   electricity_distribution_grid_cost_factor: 1.0
   electricity_grid_connection: true
+  transmission_efficiency:
+    DC:
+      efficiency_static: 0.98
+      efficiency_per_1000km: 0.977
+    H2 pipeline:
+      efficiency_per_1000km: 1 # 0.982
+      compression_per_1000km: 0.018
+    gas pipeline:
+      efficiency_per_1000km: 1 #0.977
+      compression_per_1000km: 0.01
   H2_network: true
   gas_network: false
   H2_retrofit: false
@@ -560,9 +623,43 @@ industry:
   MWh_NH3_per_MWh_H2_cracker: 1.46 # https://github.com/euronion/trace/blob/44a5ff8401762edbef80eff9cfe5a47c8d3c8be4/data/efficiencies.csv
   NH3_process_emissions: 24.5
   petrochemical_process_emissions: 25.5
-  HVC_primary_fraction: 1.
-  HVC_mechanical_recycling_fraction: 0.
-  HVC_chemical_recycling_fraction: 0.
+  #HVC primary/recycling based on values used in Neumann et al https://doi.org/10.1016/j.joule.2023.06.016, linearly interpolated between 2020 and 2050
+  #2020 recycling rates based on Agora https://static.agora-energiewende.de/fileadmin/Projekte/2021/2021_02_EU_CEAP/A-EW_254_Mobilising-circular-economy_study_WEB.pdf
+  #fractions refer to the total primary HVC production in 2020
+  #assumes 6.7 Mtplastics produced from recycling in 2020
+  HVC_primary_fraction:
+    2020: 1.0
+    2025: 0.9
+    2030: 0.8
+    2035: 0.7
+    2040: 0.6
+    2045: 0.5
+    2050: 0.4
+  HVC_mechanical_recycling_fraction:
+    2020: 0.12
+    2025: 0.15
+    2030: 0.18
+    2035: 0.21
+    2040: 0.24
+    2045: 0.27
+    2050: 0.30
+  HVC_chemical_recycling_fraction:
+    2020: 0.0
+    2025: 0.0
+    2030: 0.04
+    2035: 0.08
+    2040: 0.12
+    2045: 0.16
+    2050: 0.20
+  sector_ratios_fraction_future:
+    2020: 0.0
+    2025: 0.1
+    2030: 0.3
+    2035: 0.5
+    2040: 0.7
+    2045: 0.9
+    2050: 1.0
+  basic_chemicals_without_NH3_production_today: 69. #Mt/a, = 86 Mtethylene-equiv - 17 MtNH3
   HVC_production_today: 52.
   MWh_elec_per_tHVC_mechanical_recycling: 0.547
   MWh_elec_per_tHVC_chemical_recycling: 6.9
@@ -579,7 +676,7 @@ industry:
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#costs
 costs:
   year: 2030
-  version: v0.6.0
+  version: v0.8.1
   rooftop_share: 0.14  # based on the potentials, assuming  (0.1 kW/m2 and 10 m2/person)
   social_discountrate: 0.02
   fill_values:
@@ -605,7 +702,9 @@ costs:
     battery: 0.
     battery inverter: 0.
   emission_prices:
+    enable: false
     co2: 0.
+    co2_monthly_prices: false
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#clustering
 clustering:
@@ -627,6 +726,14 @@ clustering:
       committable: any
       ramp_limit_up: max
       ramp_limit_down: max
+  temporal:
+    resolution_elec: false
+    resolution_sector: false
+
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#adjustments
+adjustments:
+  electricity: false
+  sector: false
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#solving
 solving:
@@ -638,14 +745,22 @@ solving:
     skip_iterations: true
     rolling_horizon: false
     seed: 123
+    custom_extra_functionality: "../data/custom_extra_functionality.py"
+    # io_api: "direct"  # Increases performance but only supported for the highs and gurobi solvers
     # options that go into the optimize function
     track_iterations: false
     min_iterations: 4
     max_iterations: 6
-    transmission_losses: 0
+    transmission_losses: 2
     linearized_unit_commitment: true
     horizon: 365
 
+  constraints:
+    CCL: false
+    EQ: false
+    BAU: false
+    SAFE: false
+
   solver:
     name: gurobi
     options: gurobi-default
@@ -700,6 +815,10 @@ solving:
       solutiontype: 2 # non basic solution, ie no crossover
       barrier.convergetol: 1.e-5
       feasopt.tolerance: 1.e-6
+    copt-default:
+      Threads: 8
+      LpMethod: 2
+      Crossover: 0
     cbc-default: {} # Used in CI
     glpk-default: {} # Used in CI
 
@@ -729,6 +848,13 @@ plotting:
     color_geomap:
       ocean: white
       land: white
+  projection:
+    name: "EqualEarth"
+    # See https://scitools.org.uk/cartopy/docs/latest/reference/projections.html for alternatives, for example:
+    # name: "LambertConformal"
+    # central_longitude: 10.
+    # central_latitude: 50.
+    # standard_parallels: [35, 65]
   eu_node_location:
     x: -5.5
     y: 46.
@@ -775,7 +901,6 @@ plotting:
     hydroelectricity: '#298c81'
     PHS: '#51dbcc'
     hydro+PHS: "#08ad97"
-    wave: '#a7d4cf'
     # solar
     solar: "#f9d002"
     solar PV: "#f9d002"
@@ -924,9 +1049,11 @@ plotting:
     air heat pump: '#36eb41'
     residential urban decentral air heat pump: '#48f74f'
     services urban decentral air heat pump: '#5af95d'
+    services rural air heat pump: '#5af95d'
     urban central air heat pump: '#6cfb6b'
     ground heat pump: '#2fb537'
     residential rural ground heat pump: '#48f74f'
+    residential rural air heat pump: '#48f74f'
     services rural ground heat pump: '#5af95d'
     Ambient: '#98eb9d'
     CHP: '#8a5751'
@@ -988,6 +1115,7 @@ plotting:
     CO2 sequestration: '#f29dae'
     DAC: '#ff5270'
     co2 stored: '#f2385a'
+    co2 sequestered: '#f2682f'
     co2: '#f29dae'
     co2 vent: '#ffd4dc'
     CO2 pipeline: '#f5627f'

config/config.entsoe-all.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 

config/config.perfect.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 run:
@@ -19,13 +19,16 @@ scenario:
   opts:
   - ''
   sector_opts:
-  - 1p5-4380H-T-H-B-I-A-solar+p3-dist1
-  - 1p7-4380H-T-H-B-I-A-solar+p3-dist1
-  - 2p0-4380H-T-H-B-I-A-solar+p3-dist1
+  - 1p5-4380H-T-H-B-I-A-dist1
+  - 1p7-4380H-T-H-B-I-A-dist1
+  - 2p0-4380H-T-H-B-I-A-dist1
   planning_horizons:
   - 2020
+  - 2025
   - 2030
+  - 2035
   - 2040
+  - 2045
   - 2050
 
 

config/config.validation.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 run:

config/create_scenarios.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2023-2024 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+# This script helps to generate a scenarios.yaml file for PyPSA-Eur.
+# You can modify the template to your needs and define all possible combinations of config values that should be considered.
+
+if "snakemake" in globals():
+    filename = snakemake.output[0]
+else:
+    filename = "../config/scenarios.yaml"
+
+import itertools
+
+# Insert your config values that should be altered in the template.
+# Change `config_section` and `config_section2` to the actual config sections.
+template = """
+scenario{scenario_number}:
+    config_section:
+        config_key: {config_value}
+    config_section2:
+        config_key2: {config_value2}
+"""
+
+# Define all possible combinations of config values.
+# This must define all config values that are used in the template.
+config_values = dict(config_value=["true", "false"], config_value2=[1, 2, 3, 4])
+
+combinations = [
+    dict(zip(config_values.keys(), values))
+    for values in itertools.product(*config_values.values())
+]
+
+with open(filename, "w") as f:
+    for i, config in enumerate(combinations):
+        f.write(template.format(scenario_number=i, **config))

config/scenarios.template.yaml

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+# This file is used to define the scenarios that are run by snakemake. Each entry on the first level is a scenario. Each scenario can contain configuration overrides with respect to the config/config.yaml settings.
+#
+# Example
+#
+# custom-scenario: # name of the scenario
+#   electricity:
+#       renewable_carriers: [wind, solar] # override the list of renewable carriers
+
+normal:
+  electricity:
+    renewable_carriers:
+    - solar
+    - onwind
+    - offwind-ac
+    - offwind-dc
+    - hydro
+
+no-offwind:
+  electricity:
+    renewable_carriers:
+    - solar
+    - onwind
+    - hydro

config/test/config.electricity.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 
@@ -8,7 +8,7 @@ tutorial: true
 run:
   name: "test-elec" # use this to keep track of runs with different settings
   disable_progressbar: true
-  shared_resources: true
+  shared_resources: "test"
   shared_cutouts: true
 
 scenario:
@@ -17,7 +17,7 @@ scenario:
   clusters:
   - 5
   opts:
-  - Co2L-24H
+  - Co2L-24h
 
 countries: ['BE']
 

config/test/config.myopic.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 
@@ -7,7 +7,7 @@ tutorial: true
 run:
   name: "test-sector-myopic"
   disable_progressbar: true
-  shared_resources: true
+  shared_resources: "test"
   shared_cutouts: true
 
 foresight: myopic
@@ -20,7 +20,7 @@ scenario:
   clusters:
   - 5
   sector_opts:
-  - 24H-T-H-B-I-A-solar+p3-dist1
+  - 24h-T-H-B-I-A-dist1
   planning_horizons:
   - 2030
   - 2040

config/test/config.overnight.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 
@@ -7,7 +7,7 @@ tutorial: true
 run:
   name: "test-sector-overnight"
   disable_progressbar: true
-  shared_resources: true
+  shared_resources: "test"
   shared_cutouts: true
 
 
@@ -19,7 +19,7 @@ scenario:
   clusters:
   - 5
   sector_opts:
-  - CO2L0-24H-T-H-B-I-A-solar+p3-dist1
+  - CO2L0-24h-T-H-B-I-A-dist1
   planning_horizons:
   - 2030
 

config/test/config.perfect.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 
@@ -7,7 +7,7 @@ tutorial: true
 run:
   name: "test-sector-perfect"
   disable_progressbar: true
-  shared_resources: true
+  shared_resources: "test"
   shared_cutouts: true
 
 foresight: perfect
@@ -18,7 +18,7 @@ scenario:
   clusters:
   - 5
   sector_opts:
-  - 8760H-T-H-B-I-A-solar+p3-dist1
+  - 8760h-T-H-B-I-A-dist1
   planning_horizons:
   - 2030
   - 2040

config/test/config.scenarios.yaml

@@ -0,0 +1,60 @@
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: CC0-1.0
+
+tutorial: true
+
+run:
+  name:
+  - test-elec-no-offshore-wind
+  - test-elec-no-onshore-wind
+  scenarios:
+    enable: true
+    file: "config/test/scenarios.yaml"
+  disable_progressbar: true
+  shared_resources: base
+  shared_cutouts: true
+
+scenario:
+  clusters:
+  - 5
+  opts:
+  - Co2L-24H
+
+countries: ['BE']
+
+snapshots:
+  start: "2013-03-01"
+  end: "2013-03-08"
+
+electricity:
+  extendable_carriers:
+    Generator: [OCGT]
+    StorageUnit: [battery, H2]
+    Store: []
+
+atlite:
+  default_cutout: be-03-2013-era5
+  cutouts:
+    be-03-2013-era5:
+      module: era5
+      x: [4., 15.]
+      y: [46., 56.]
+      time: ["2013-03-01", "2013-03-08"]
+
+renewable:
+  onwind:
+    cutout: be-03-2013-era5
+  offwind-ac:
+    cutout: be-03-2013-era5
+    max_depth: false
+  offwind-dc:
+    cutout: be-03-2013-era5
+    max_depth: false
+  solar:
+    cutout: be-03-2013-era5
+
+solving:
+  solver:
+    name: glpk
+    options: "glpk-default"

config/test/scenarios.yaml

@@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: CC0-1.0
+
+test-elec-no-offshore-wind:
+  electricity:
+    renewable_carriers: [solar, onwind]
+
+test-elec-no-onshore-wind:
+  electricity:
+    renewable_carriers: [solar, offwind-ac, offwind-dc]

data/custom_extra_functionality.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2023- The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+
+def custom_extra_functionality(n, snapshots, snakemake):
+    """
+    Add custom extra functionality constraints.
+    """
+    pass

data/switzerland-new_format-all_years.csv

@@ -0,0 +1,25 @@
+country,item,2010,2011,2012,2013,2014,2015
+CH,total residential,268.2,223.4,243.4,261.3,214.2,229.1
+CH,total residential space,192.2,149.0,168.1,185.5,139.7,154.4
+CH,total residential water,32.2,31.6,31.9,32.2,31.7,31.9
+CH,total residential cooking,9.3,9.3,9.3,9.4,9.5,9.6
+CH,electricity residential,67.9,63.7,65.7,67.6,63.0,64.4
+CH,electricity residential space,15.9,12.8,14.3,15.8,12.3,13.5
+CH,electricity residential water,8.8,8.5,8.5,8.6,8.5,8.6
+CH,electricity residential cooking,4.9,4.9,4.9,4.9,5.0,5.0
+CH,total services,145.9,127.4,136.7,144.0,124.5,132.5
+CH,total services space,80.0,62.2,70.8,77.4,58.3,64.3
+CH,total services water,10.1,10.0,10.1,10.1,10.0,10.0
+CH,total services cooking,2.5,2.4,2.3,2.3,2.4,2.3
+CH,electricity services,60.5,59.2,60.3,61.4,60.3,62.6
+CH,electricity services space,4.0,3.2,3.8,4.2,3.3,3.6
+CH,electricity services water,0.7,0.7,0.7,0.7,0.7,0.7
+CH,electricity services cooking,2.5,2.4,2.3,2.3,2.4,2.3
+CH,total rail,11.5,11.1,11.2,11.4,11.1,11.4
+CH,total road,199.4,200.4,200.4,201.2,202.0,203.1
+CH,electricity road,0.,0.,0.,0.,0.,0.
+CH,electricity rail,11.5,11.1,11.2,11.4,11.1,11.4
+CH,total domestic aviation,3.3,3.2,3.4,3.4,3.5,3.5
+CH,total international aviation,58.0,62.0,63.5,64.2,64.5,66.8
+CH,total domestic navigation,1.6,1.6,1.6,1.6,1.6,1.6
+CH,total international navigation,0.,0.,0.,0.,0.,0.

doc/Makefile

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 

doc/conf.py

@@ -72,7 +72,7 @@ master_doc = "index"
 
 # General information about the project.
 project = "PyPSA-Eur"
-copyright = "2017-2023 Tom Brown (KIT, TUB, FIAS), Jonas Hoersch (KIT, FIAS), Fabian Hofmann (TUB, FIAS), Fabian Neumann (TUB, KIT), Marta Victoria (Aarhus University), Lisa Zeyen (KIT, TUB)"
+copyright = "2017-2024 Tom Brown (KIT, TUB, FIAS), Jonas Hoersch (KIT, FIAS), Fabian Hofmann (TUB, FIAS), Fabian Neumann (TUB, KIT), Marta Victoria (Aarhus University), Lisa Zeyen (KIT, TUB)"
 author = "Tom Brown (KIT, TUB, FIAS), Jonas Hoersch (KIT, FIAS), Fabian Hofmann (TUB, FIAS), Fabian Neumann (TUB, KIT), Marta Victoria (Aarhus University), Lisa Zeyen (KIT, TUB)"
 
 # The version info for the project you're documenting, acts as replacement for
@@ -80,9 +80,9 @@ author = "Tom Brown (KIT, TUB, FIAS), Jonas Hoersch (KIT, FIAS), Fabian Hofmann
 # built documents.
 #
 # The short X.Y version.
-version = "0.8"
+version = "0.10"
 # The full version, including alpha/beta/rc tags.
-release = "0.8.1"
+release = "0.10.0"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

doc/configtables/adjustments.csv

@@ -0,0 +1,8 @@
+,Unit,Values,Description
+adjustments,,,
+-- electricity,bool or dict,,"Parameter adjustments for capital cost, marginal cost, and maximum capacities of carriers. Applied in :mod:`prepare_network.`"
+-- -- {attr},,,"Attribute can be ``e_nom_opt``, ``p_nom_opt``, ``marginal_cost`` or ``capital_cost``"
+-- -- -- {carrier},float,per-unit,"Any carrier of the network to which parameter adjustment factor should be applied."
+-- sector,bool or dict,,"Parameter adjustments for capital cost, marginal cost, and maximum capacities of carriers. Applied in :mod:`prepare_sector_network.`"
+-- -- {attr},,,"Attribute can be ``e_nom_opt``, ``p_nom_opt``, ``marginal_cost`` or ``capital_cost``"
+-- -- -- {carrier},float,per-unit,"Any carrier of the network to which parameter adjustment factor should be applied."

doc/configtables/clustering.csv

@@ -17,3 +17,6 @@ aggregation_strategies,,,
 -- -- {key},str,"{key} can be any of the component of the generator (str). It’s value can be any that can be converted to pandas.Series using getattr(). For example one of {min, max, sum}.","Aggregates the component according to the given strategy. For example, if sum, then all values within each cluster are summed to represent the new generator."
 -- buses,,,
 -- -- {key},str,"{key} can be any of the component of the bus (str). It’s value can be any that can be converted to pandas.Series using getattr(). For example one of {min, max, sum}.","Aggregates the component according to the given strategy. For example, if sum, then all values within each cluster are summed to represent the new bus."
+temporal,,,Options for temporal resolution
+-- resolution_elec,--,"{false,``nH``; i.e. ``2H``-``6H``}","Resample the time-resolution by averaging over every ``n`` snapshots in :mod:`prepare_network`. **Warning:** This option should currently only be used with electricity-only networks, not for sector-coupled networks."
+-- resolution_sector,--,"{false,``nH``; i.e. ``2H``-``6H``}","Resample the time-resolution by averaging over every ``n`` snapshots in :mod:`prepare_sector_network`."

doc/configtables/costs.csv

@@ -1,9 +1,12 @@
-,Unit,Values,Description
-year,--,"YYYY; e.g. '2030'","Year for which to retrieve cost assumptions of ``resources/costs.csv``."
-version,--,"vX.X.X; e.g. 'v0.5.0'","Version of ``technology-data`` repository to use."
-rooftop_share,--,float,"Share of rooftop PV when calculating capital cost of solar (joint rooftop and utility-scale PV)."
-fill_values,--,float,"Default values if not specified for a technology in ``resources/costs.csv``."
-capital_cost,EUR/MW,"Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.","For the given technologies, assumptions about their capital investment costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``."
-marginal_cost,EUR/MWh,"Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.","For the given technologies, assumptions about their marginal operating costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``."
-emission_prices,,,"Specify exogenous prices for emission types listed in ``network.carriers`` to marginal costs."
--- co2,EUR/t,float,"Exogenous price of carbon-dioxide added to the marginal costs of fossil-fuelled generators according to their carbon intensity. Added through the keyword ``Ep`` in the ``{opts}`` wildcard only in the rule :mod:`prepare_network``."
+,Unit,Values,Description
+year,--,YYYY; e.g. '2030',Year for which to retrieve cost assumptions of ``resources/costs.csv``.
+version,--,vX.X.X; e.g. 'v0.5.0',Version of ``technology-data`` repository to use.
+rooftop_share,--,float,Share of rooftop PV when calculating capital cost of solar (joint rooftop and utility-scale PV).
+social_discountrate,p.u.,float,Social discount rate to compare costs in different investment periods. 0.02 corresponds to a social discount rate of 2%.
+fill_values,--,float,Default values if not specified for a technology in ``resources/costs.csv``.
+capital_cost,EUR/MW,Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.,"For the given technologies, assumptions about their capital investment costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``."
+marginal_cost,EUR/MWh,Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.,"For the given technologies, assumptions about their marginal operating costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``."
+emission_prices,,,Specify exogenous prices for emission types listed in ``network.carriers`` to marginal costs.
+-- enable,bool,true or false,Add cost for a carbon-dioxide price configured in ``costs: emission_prices: co2`` to ``marginal_cost`` of generators (other emission types listed in ``network.carriers`` possible as well)
+-- co2,EUR/t,float,Exogenous price of carbon-dioxide added to the marginal costs of fossil-fuelled generators according to their carbon intensity. Added through the keyword ``Ep`` in the ``{opts}`` wildcard only in the rule :mod:`prepare_network``.
+-- co2_monthly_price,bool,true or false,Add monthly cost for a carbon-dioxide price based on historical values built by the rule ``build_monthly_prices``

doc/configtables/electricity.csv

@@ -1,6 +1,8 @@
 ,Unit,Values,Description
 voltages,kV,"Any subset of {220., 300., 380.}",Voltage levels to consider
+gaslimit_enable,bool,true or false,Add an overall absolute gas limit configured in ``electricity: gaslimit``.
 gaslimit,MWhth,float or false,Global gas usage limit
+co2limit_enable,bool,true or false,Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``.
 co2limit,:math:`t_{CO_2-eq}/a`,float,Cap on total annual system carbon dioxide emissions
 co2base,:math:`t_{CO_2-eq}/a`,float,Reference value of total annual system carbon dioxide emissions if relative emission reduction target is specified in ``{opts}`` wildcard.
 agg_p_nom_limits,file,path,Reference to ``.csv`` file specifying per carrier generator nominal capacity constraints for individual countries if ``'CCL'`` is in ``{opts}`` wildcard. Defaults to ``data/agg_p_nom_minmax.csv``.
@@ -22,6 +24,8 @@ powerplants_filter,--,"use `pandas.query <https://pandas.pydata.org/pandas-docs/
 ,,,
 custom_powerplants,--,"use `pandas.query <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.query.html>`_ strings here, e.g. ``Country in ['Germany']``",Filter query for the custom powerplant database.
 ,,,
+everywhere_powerplants,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to add to every node in the model with zero initial capacity. To be used in combination with ``extendable_carriers`` to allow for building conventional powerplants irrespective of existing locations."
+,,,
 conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to include in the model from ``resources/powerplants.csv``. If an included carrier is also listed in ``extendable_carriers``, the capacity is taken as a lower bound."
 ,,,
 renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, hydro}",List of renewable generators to include in the model.
@@ -34,3 +38,6 @@ estimate_renewable_capacities,,,
 -- -- Offshore,--,"Any subset of {offwind-ac, offwind-dc}","List of PyPSA-Eur carriers that is considered as (IRENA, OPSD) onshore technology."
 -- -- Offshore,--,{onwind},"List of PyPSA-Eur carriers that is considered as (IRENA, OPSD) offshore technology."
 -- -- PV,--,{solar},"List of PyPSA-Eur carriers that is considered as (IRENA, OPSD) PV technology."
+autarky,,,
+-- enable,bool,true or false,Require each node to be autarkic by removing all lines and links.
+-- by_country,bool,true or false,Require each country to be autarkic by removing all cross-border lines and links. ``electricity: autarky`` must be enabled.

doc/configtables/existing_capacities.csv

@@ -3,4 +3,5 @@ grouping_years_power ,--,A list of years,Intervals to group existing capacities
 grouping_years_heat ,--,A list of years below 2020,Intervals to group existing capacities for heat
 
 threshold_capacity ,MW,float,Capacities generators and links of below threshold are removed during add_existing_capacities
+default_heating_lifetime ,years,int,Default lifetime for heating technologies
 conventional_carriers ,--,"Any subset of {uranium, coal, lignite, oil} ",List of conventional power plants to include in the sectoral network

doc/configtables/industry.csv

@@ -17,6 +17,8 @@ HVC_primary_fraction,--,float,The fraction of high value chemicals (HVC) produce
 HVC_mechanical_recycling _fraction,--,float,The fraction of high value chemicals (HVC) produced using mechanical recycling
 HVC_chemical_recycling _fraction,--,float,The fraction of high value chemicals (HVC) produced using chemical recycling
 ,,,
+sector_ratios_fraction_future,--,Dictionary with planning horizons as keys.,The fraction of total progress in fuel and process switching achieved in the industry sector.
+basic_chemicals_without_NH3_production_today,Mt/a,float,"The amount of basic chemicals produced without ammonia (= 86 Mtethylene-equiv - 17 MtNH3)."
 HVC_production_today,MtHVC/a,float,"The amount of high value chemicals (HVC) produced. This includes ethylene, propylene and BTX. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, Figure 16, page 107"
 Mwh_elec_per_tHVC _mechanical_recycling,MWh/tHVC,float,"The energy amount of electricity needed to produce a ton of high value chemical (HVC) using mechanical recycling. From SI of `Meys et al (2020) <https://doi.org/10.1016/j.resconrec.2020.105010>`_, Table S5, for HDPE, PP, PS, PET. LDPE would be 0.756."
 Mwh_elec_per_tHVC _chemical_recycling,MWh/tHVC,float,"The energy amount of electricity needed to produce a ton of high value chemical (HVC) using chemical recycling. The default value is based on pyrolysis and electric steam cracking. From `Material Economics (2019) <https://materialeconomics.com/latest-updates/industrial-transformation-2050>`_, page 125"

doc/configtables/licenses-sector.csv

@@ -9,9 +9,8 @@ Swiss energy statistics from Swiss Federal Office of Energy,switzerland-sfoe/,un
 BASt emobility statistics,emobility/,unknown,http://www.bast.de/DE/Verkehrstechnik/Fachthemen/v2-verkehrszaehlung/Stundenwerte.html?nn=626916
 BDEW heating profile,heat_load_profile_BDEW.csv,unknown,https://github.com/oemof/demandlib
 heating profiles for Aarhus,heat_load_profile_DK_AdamJensen.csv,unknown,Adam Jensen MA thesis at Aarhus University
-George Lavidas wind/wave costs,WindWaveWEC_GLTB.xlsx,unknown,George Lavidas
 co2 budgets,co2_budget.csv,CC BY 4.0,https://arxiv.org/abs/2004.11009
-existing heating potentials,existing_infrastructure/existing_heating_raw.csv,unknown,https://ec.europa.eu/energy/studies/mapping-and-analyses-current-and-future-2020-2030-heatingcooling-fuel-deployment_en?redir=1
+existing heating potentials,existing_infrastructure/existing_heating_raw.csv,unknown,https://energy.ec.europa.eu/publications/mapping-and-analyses-current-and-future-2020-2030-heatingcooling-fuel-deployment-fossilrenewables-1_en
 IRENA existing VRE capacities,existing_infrastructure/{solar|onwind|offwind}_capcity_IRENA.csv,unknown,https://www.irena.org/Statistics/Download-Data
 USGS ammonia production,myb1-2017-nitro.xls,unknown,https://www.usgs.gov/centers/nmic/nitrogen-statistics-and-information
 hydrogen salt cavern potentials,h2_salt_caverns_GWh_per_sqkm.geojson,CC BY 4.0,https://doi.org/10.1016/j.ijhydene.2019.12.161 https://doi.org/10.20944/preprints201910.0187.v1

doc/configtables/load.csv

@@ -1,8 +1,5 @@
 ,Unit,Values,Description
-<<<<<<< HEAD
 url,--,string,"Link to open power system data time series data."
-=======
->>>>>>> master
 power_statistics,bool,"{true, false}",Whether to load the electricity consumption data of the ENTSOE power statistics (only for files from 2019 and before) or from the ENTSOE transparency data (only has load data from 2015 onwards).
 interpolate_limit,hours,integer,"Maximum gap size (consecutive nans) which interpolated linearly."
 time_shift_for_large_gaps,string,string,"Periods which are used for copying time-slices in order to fill large gaps of nans. Have to be valid ``pandas`` period strings."

doc/configtables/offwind-ac.csv

@@ -2,11 +2,12 @@
 cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored."
 resource,,,
 -- method,--,"Must be 'wind'","A superordinate technology type."
--- turbine,--,"One of turbine types included in `atlite <https://github.com/PyPSA/atlite/tree/master/atlite/resources/windturbine>`_","Specifies the turbine type and its characteristic power curve."
+-- turbine,--,"One of turbine types included in `atlite <https://github.com/PyPSA/atlite/tree/master/atlite/resources/windturbine>`_.  Can be a string or a dictionary with years as keys which denote the year another turbine model becomes available.","Specifies the turbine type and its characteristic power curve."
 capacity_per_sqkm,:math:`MW/km^2`,float,"Allowable density of wind turbine placement."
 correction_factor,--,float,"Correction factor for capacity factor time series."
 excluder_resolution,m,float,"Resolution on which to perform geographical elibility analysis."
 corine,--,"Any *realistic* subset of the `CORINE Land Cover code list <http://www.eea.europa.eu/data-and-maps/data/corine-land-cover-2006-raster-1/corine-land-cover-classes-and/clc_legend.csv/at_download/file>`_","Specifies areas according to CORINE Land Cover codes which are generally eligible for AC-connected offshore wind turbine placement."
+luisa,--,"Any subset of the `LUISA Base Map codes in Annex 1 <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_","Specifies areas according to the LUISA Base Map codes which are generally eligible for AC-connected offshore wind turbine placement."
 natura,bool,"{true, false}","Switch to exclude `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas. Area is excluded if ``true``."
 ship_threshold,--,float,"Ship density threshold from which areas are excluded."
 max_depth,m,float,"Maximum sea water depth at which wind turbines can be build. Maritime areas with deeper waters are excluded in the process of calculating the AC-connected offshore wind potential."

doc/configtables/offwind-dc.csv

@@ -2,11 +2,12 @@
 cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored."
 resource,,,
 -- method,--,"Must be 'wind'","A superordinate technology type."
--- turbine,--,"One of turbine types included in `atlite <https://github.com/PyPSA/atlite/tree/master/atlite/resources/windturbine>`__","Specifies the turbine type and its characteristic power curve."
+-- turbine,--,"One of turbine types included in `atlite <https://github.com/PyPSA/atlite/tree/master/atlite/resources/windturbine>`_. Can be a string or a dictionary with years as keys which denote the year another turbine model becomes available.","Specifies the turbine type and its characteristic power curve."
 capacity_per_sqkm,:math:`MW/km^2`,float,"Allowable density of wind turbine placement."
 correction_factor,--,float,"Correction factor for capacity factor time series."
 excluder_resolution,m,float,"Resolution on which to perform geographical elibility analysis."
 corine,--,"Any *realistic* subset of the `CORINE Land Cover code list <http://www.eea.europa.eu/data-and-maps/data/corine-land-cover-2006-raster-1/corine-land-cover-classes-and/clc_legend.csv/at_download/file>`_","Specifies areas according to CORINE Land Cover codes which are generally eligible for AC-connected offshore wind turbine placement."
+luisa,--,"Any subset of the `LUISA Base Map codes in Annex 1 <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_","Specifies areas according to the LUISA Base Map codes which are generally eligible for DC-connected offshore wind turbine placement."
 natura,bool,"{true, false}","Switch to exclude `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas. Area is excluded if ``true``."
 ship_threshold,--,float,"Ship density threshold from which areas are excluded."
 max_depth,m,float,"Maximum sea water depth at which wind turbines can be build. Maritime areas with deeper waters are excluded in the process of calculating the AC-connected offshore wind potential."

doc/configtables/onwind.csv

@@ -2,12 +2,16 @@
 cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored."
 resource,,,
 -- method,--,"Must be 'wind'","A superordinate technology type."
--- turbine,--,"One of turbine types included in `atlite <https://github.com/PyPSA/atlite/tree/master/atlite/resources/windturbine>`__","Specifies the turbine type and its characteristic power curve."
+-- turbine,--,"One of turbine types included in `atlite <https://github.com/PyPSA/atlite/tree/master/atlite/resources/windturbine>`_. Can be a string or a dictionary with years as keys which denote the year another turbine model becomes available.","Specifies the turbine type and its characteristic power curve."
 capacity_per_sqkm,:math:`MW/km^2`,float,"Allowable density of wind turbine placement."
 corine,,,
 -- grid_codes,--,"Any subset of the `CORINE Land Cover code list <http://www.eea.europa.eu/data-and-maps/data/corine-land-cover-2006-raster-1/corine-land-cover-classes-and/clc_legend.csv/at_download/file>`_","Specifies areas according to CORINE Land Cover codes which are generally eligible for wind turbine placement."
 -- distance,m,float,"Distance to keep from areas specified in ``distance_grid_codes``"
 -- distance_grid_codes,--,"Any subset of the `CORINE Land Cover code list <http://www.eea.europa.eu/data-and-maps/data/corine-land-cover-2006-raster-1/corine-land-cover-classes-and/clc_legend.csv/at_download/file>`_","Specifies areas according to CORINE Land Cover codes to which wind turbines must maintain a distance specified in the setting ``distance``."
+luisa,,,
+-- grid_codes,--,"Any subset of the `LUISA Base Map codes in Annex 1 <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_","Specifies areas according to the LUISA Base Map codes which are generally eligible for wind turbine placement."
+-- distance,m,float,"Distance to keep from areas specified in ``distance_grid_codes``"
+-- distance_grid_codes,--,"Any subset of the `LUISA Base Map codes in Annex 1 <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_","Specifies areas according to the LUISA Base Map codes to which wind turbines must maintain a distance specified in the setting ``distance``."
 natura,bool,"{true, false}","Switch to exclude `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas. Area is excluded if ``true``."
 clip_p_max_pu,p.u.,float,"To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero."
 correction_factor,--,float,"Correction factor for capacity factor time series."

doc/configtables/opts.csv

@@ -1,13 +1,13 @@
-Trigger, Description, Definition, Status
-``nH``; i.e. ``2H``-``6H``, Resample the time-resolution by averaging over every ``n`` snapshots, ``prepare_network``: `average_every_nhours() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L110>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L146>`__), In active use
-``nSEG``; e.g. ``4380SEG``, "Apply time series segmentation with `tsam <https://tsam.readthedocs.io/en/latest/index.html>`_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load.", ``prepare_network``: apply_time_segmentation(), In active use
-``Co2L``, Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``. If a float is appended an overall emission limit relative to the emission level given in ``electricity: co2base`` is added (e.g. ``Co2L0.05`` limits emissisions to 5% of what is given in ``electricity: co2base``), ``prepare_network``: `add_co2limit() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L19>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L154>`__, In active use
-``Ep``, Add cost for a carbon-dioxide price configured in ``costs: emission_prices: co2`` to ``marginal_cost`` of generators (other emission types listed in ``network.carriers`` possible as well), ``prepare_network``: `add_emission_prices() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L24>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L158>`__, In active use
-``Ept``, Add monthly cost for a carbon-dioxide price based on historical values built by the rule ``build_monthly_prices``, In active use
-``CCL``, Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``., ``solve_network``, In active use
-``EQ``, "Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption.", ``solve_network``, In active use
-``ATK``, "Require each node to be autarkic. Example: ``ATK`` removes all lines and links. ``ATKc`` removes all cross-border lines and links.", ``prepare_network``, In active use
-``BAU``, Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities``, ``solve_network``: `add_opts_constraints() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/solve_network.py#L66>`__, Untested
-``SAFE``, Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network., ``solve_network`` `add_opts_constraints() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/solve_network.py#L73>`__, Untested
-``carrier+{c|p|m}factor``,"Alter the capital cost (``c``), installable potential (``p``) or marginal costs (``m``) of a carrier by a factor. Example: ``solar+c0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use
-``CH4L``,"Add an overall absolute gas limit. If configured in ``electricity: gaslimit`` it is given in MWh thermal, if a float is appended, the overall gaslimit is assumed to be given in TWh thermal (e.g. ``CH4L200`` limits gas dispatch to 200 TWh termal)", ``prepare_network``: ``add_gaslimit()``, In active use
+Trigger, Description, Definition, Status
+``nH``; i.e. ``2H``-``6H``, Resample the time-resolution by averaging over every ``n`` snapshots, ``prepare_network``: `average_every_nhours() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L110>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L146>`__), In active use
+``nSEG``; e.g. ``4380SEG``,"Apply time series segmentation with `tsam <https://tsam.readthedocs.io/en/latest/index.html>`_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load.", ``prepare_network``: apply_time_segmentation(), In active use
+``Co2L``,Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``. If a float is appended an overall emission limit relative to the emission level given in ``electricity: co2base`` is added (e.g. ``Co2L0.05`` limits emissisions to 5% of what is given in ``electricity: co2base``), ``prepare_network``: `add_co2limit() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L19>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L154>`__, In active use
+``Ep``,Add cost for a carbon-dioxide price configured in ``costs: emission_prices: co2`` to ``marginal_cost`` of generators (other emission types listed in ``network.carriers`` possible as well), ``prepare_network``: `add_emission_prices() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L24>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L158>`__, In active use
+``Ept``,Add monthly cost for a carbon-dioxide price based on historical values built by the rule ``build_monthly_prices``, In active use,
+``CCL``,Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``., ``solve_network``, In active use
+``EQ``,Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption., ``solve_network``, In active use
+``ATK``,Require each node to be autarkic. Example: ``ATK`` removes all lines and links. ``ATKc`` removes all cross-border lines and links., ``prepare_network``, In active use
+``BAU``,Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities``, ``solve_network``: `add_opts_constraints() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/solve_network.py#L66>`__, Untested
+``SAFE``,Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network., ``solve_network`` `add_opts_constraints() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/solve_network.py#L73>`__, Untested
+``carrier+{c|p|m}factor``,"Alter the capital cost (``c``), installable potential (``p``) or marginal costs (``m``) of a carrier by a factor. Example: ``solar+c0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use
+``CH4L``,"Add an overall absolute gas limit. If configured in ``electricity: gaslimit`` it is given in MWh thermal, if a float is appended, the overall gaslimit is assumed to be given in TWh thermal (e.g. ``CH4L200`` limits gas dispatch to 200 TWh termal)", ``prepare_network``: ``add_gaslimit()``, In active use

doc/configtables/plotting.csv

@@ -1,6 +1,9 @@
 ,Unit,Values,Description
 map,,,
 -- boundaries,°,"[x1,x2,y1,y2]",Boundaries of the map plots in degrees latitude (y) and longitude (x)
+projection,,,,
+-- name,--,"Valid Cartopy projection name","See https://scitools.org.uk/cartopy/docs/latest/reference/projections.html for list of available projections."
+-- args,--,--,"Other entries under 'projection' are passed as keyword arguments to the projection constructor, e.g. ``central_longitude: 10.``."
 costs_max,bn Euro,float,Upper y-axis limit in cost bar plots.
 costs_threshold,bn Euro,float,Threshold below which technologies will not be shown in cost bar plots.
 energy_max,TWh,float,Upper y-axis limit in energy bar plots.

doc/configtables/run.csv

@@ -1,5 +1,8 @@
 ,Unit,Values,Description
-name,--,"any string","Specify a name for your run. Results will be stored under this name."
-disable_progrssbar,bool,"{true, false}","Switch to select whether progressbar should be disabled."
-shared_resources,bool,"{true, false}","Switch to select whether resources should be shared across runs."
+name,--,str/list,"Specify a name for your run. Results will be stored under this name. If ``scenario: enable:`` is set to ``true``, the name must contain a subset of scenario names defined in ``scenario: file:``. If the name is 'all', all defined scenarios will be run."
+scenarios,,,
+-- enable,bool,"{true, false}","Switch to select whether workflow should generate scenarios based on ``file``."
+-- file,str,,"Path to the scenario yaml file. The scenario file contains config overrides for each scenario. In order to be taken account, ``run: scenarios`` has to be set to ``true`` and ``run: name`` has to be a subset of top level keys given in the scenario file. In order to automatically create a `scenario.yaml` file based on a combination of settings, alter and use the ``config/create_scenarios.py`` script in the ``config`` directory."
+disable_progressbar,bool,"{true, false}","Switch to select whether progressbar should be disabled."
+shared_resources,bool/str,,"Switch to select whether resources should be shared across runs. If a string is passed, this is used as a subdirectory name for shared resources. If set to 'base', only resources before creating the elec.nc file are shared."
 shared_cutouts,bool,"{true, false}","Switch to select whether cutouts should be shared across runs."

doc/configtables/sector-opts.csv

@@ -7,5 +7,5 @@ Trigger, Description, Definition, Status
 ``B``,Add biomass,,In active use
 ``I``,Add industry sector,,In active use
 ``A``,Add agriculture sector,,In active use
-``dist``+``n``,Add distribution grid with investment costs of ``n`` times costs in ``data/costs_{cost_year}.csv``,,In active use
+``dist``+``n``,Add distribution grid with investment costs of ``n`` times costs in ``resources/costs_{cost_year}.csv``,,In active use
 ``seq``+``n``,Sets the CO2 sequestration potential to ``n`` Mt CO2 per year,,In active use

doc/configtables/sector.csv

@@ -1,4 +1,9 @@
 ,Unit,Values,Description
+transport,--,"{true, false}",Flag to include transport sector.
+heating,--,"{true, false}",Flag to include heating sector.
+biomass,--,"{true, false}",Flag to include biomass sector.
+industry,--,"{true, false}",Flag to include industry sector.
+agriculture,--,"{true, false}",Flag to include agriculture sector.
 district_heating,--,,`prepare_sector_network.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/prepare_sector_network.py>`_
 -- potential,--,float,maximum fraction of urban demand which can be supplied by district heating
 -- progress,--,Dictionary with planning horizons as keys., Increase of today's district heating demand to potential maximum district heating share. Progress = 0 means today's district heating share. Progress = 1 means maximum fraction of urban demand is supplied by district heating
@@ -62,9 +67,11 @@ tes,--,"{true, false}",Add option for storing thermal energy in large water pits
 tes_tau,,,The time constant used to calculate the decay of thermal energy in thermal energy storage (TES): 1- :math:`e^{-1/24τ}`.
 -- decentral,days,float,The time constant in decentralized thermal energy storage (TES)
 -- central,days,float,The time constant in centralized thermal energy storage (TES)
-boilers,--,"{true, false}",Add option for transforming electricity into heat using resistive heater
+boilers,--,"{true, false}",Add option for transforming gas into heat using gas boilers
+resistive_heaters,--,"{true, false}",Add option for transforming electricity into heat using resistive heaters (independently from gas boilers)
 oil_boilers,--,"{true, false}",Add option for transforming oil into heat using boilers
 biomass_boiler,--,"{true, false}",Add option for transforming biomass into heat using boilers
+overdimension_individual_heating,--,"float",Add option for overdimensioning individual heating systems by a certain factor. This allows them to cover heat demand peaks e.g. 10% higher than those in the data with a setting of 1.1.
 chp,--,"{true, false}",Add option for using Combined Heat and Power (CHP)
 micro_chp,--,"{true, false}",Add option for using Combined Heat and Power (CHP) for decentral areas.
 solar_thermal,--,"{true, false}",Add option for using solar thermal to generate heat.
@@ -79,6 +86,8 @@ hydrogen_fuel_cell,--,"{true, false}",Add option to include hydrogen fuel cell f
 hydrogen_turbine,--,"{true, false}",Add option to include hydrogen turbine for re-electrification. Assuming OCGT technology costs
 SMR,--,"{true, false}",Add option for transforming natural gas into hydrogen and CO2 using Steam Methane Reforming (SMR)
 SMR CC,--,"{true, false}",Add option for transforming natural gas into hydrogen and CO2 using Steam Methane Reforming (SMR) and Carbon Capture (CC)
+regional_methanol_demand,--,"{true, false}",Spatially resolve methanol demand. Set to true if regional CO2 constraints needed.
+regional_oil_demand,--,"{true, false}",Spatially resolve oil demand. Set to true if regional CO2 constraints needed.
 regional_co2 _sequestration_potential,,,
 -- enable,--,"{true, false}",Add option for regionally-resolved geological carbon dioxide sequestration potentials based on `CO2StoP <https://setis.ec.europa.eu/european-co2-storage-database_en>`_.
 -- attribute,--,string,Name of the attribute for the sequestration potential
@@ -88,9 +97,11 @@ regional_co2 _sequestration_potential,,,
 -- years_of_storage,years,float,The years until potential exhausted at optimised annual rate
 co2_sequestration_potential,MtCO2/a,float,The potential of sequestering CO2 in Europe per year
 co2_sequestration_cost,currency/tCO2,float,The cost of sequestering a ton of CO2
+co2_sequestration_lifetime,years,int,The lifetime of a CO2 sequestration site
 co2_spatial,--,"{true, false}","Add option to spatially resolve carrier representing stored carbon dioxide. This allows for more detailed modelling of CCUTS, e.g. regarding the capturing of industrial process emissions, usage as feedstock for electrofuels, transport of carbon dioxide, and geological sequestration sites."
 ,,,
 co2network,--,"{true, false}",Add option for planning a new carbon dioxide transmission network
+co2_network_cost_factor,p.u.,float,The cost factor for the capital cost of the carbon dioxide transmission network
 ,,,
 cc_fraction,--,float,The default fraction of CO2 captured with post-combustion capture
 hydrogen_underground _storage,--,"{true, false}",Add options for storing hydrogen underground. Storage potential depends regionally.
@@ -103,10 +114,16 @@ min_part_load _methanolisation,per unit of p_nom ,float,The minimum unit dispatc
 use_fischer_tropsch _waste_heat,--,"{true, false}",Add option for using waste heat of Fischer Tropsch in district heating networks
 use_fuel_cell_waste_heat,--,"{true, false}",Add option for using waste heat of fuel cells in district heating networks
 use_electrolysis_waste _heat,--,"{true, false}",Add option for using waste heat of electrolysis in district heating networks
+electricity_transmission _grid,--,"{true, false}",Switch for enabling/disabling the electricity transmission grid.
 electricity_distribution _grid,--,"{true, false}",Add a simplified representation of the exchange capacity between transmission and distribution grid level through a link.
 electricity_distribution _grid_cost_factor,,,Multiplies the investment cost of the electricity distribution grid
 ,,,
 electricity_grid _connection,--,"{true, false}",Add the cost of electricity grid connection for onshore wind and solar
+transmission_efficiency,,,Section to specify transmission losses or compression energy demands of bidirectional links. Splits them into two capacity-linked unidirectional links.
+-- {carrier},--,str,The carrier of the link.
+-- -- efficiency_static,p.u.,float,Length-independent transmission efficiency.
+-- -- efficiency_per_1000km,p.u. per 1000 km,float,Length-dependent transmission efficiency ($\eta^{\text{length}}$)
+-- -- compression_per_1000km,p.u. per 1000 km,float,Length-dependent electricity demand for compression ($\eta \cdot \text{length}$) implemented as multi-link to local electricity bus.
 H2_network,--,"{true, false}",Add option for new hydrogen pipelines
 gas_network,--,"{true, false}","Add existing natural gas infrastructure, incl. LNG terminals, production and entry-points. The existing gas network is added with a lossless transport model. A length-weighted `k-edge augmentation algorithm   <https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation>`_   can be run to add new candidate gas pipelines such that all regions of the model can be connected to the gas network. When activated, all the gas demands are regionally disaggregated as well."
 H2_retrofit,--,"{true, false}",Add option for retrofiting existing pipelines to transport hydrogen.
@@ -121,3 +138,10 @@ biogas_upgrading_cc,--,"{true, false}",Add option to capture CO2 from biomass up
 conventional_generation,,,Add a more detailed description of conventional carriers. Any power generation requires the consumption of fuel from nodes representing that fuel.
 biomass_to_liquid,--,"{true, false}",Add option for transforming solid biomass into liquid fuel with the same properties as oil
 biosng,--,"{true, false}",Add option for transforming solid biomass into synthesis gas with the same properties as natural gas
+limit_max_growth,,,
+-- enable,--,"{true, false}",Add option to limit the maximum growth of a carrier
+-- factor,p.u.,float,The maximum growth factor of a carrier (e.g. 1.3 allows  30% larger than max historic growth)
+-- max_growth,,,
+-- -- {carrier},GW,float,The historic maximum growth of a carrier
+-- max_relative_growth,
+-- -- {carrier},p.u.,float,The historic maximum relative growth of a carrier

doc/configtables/snapshots.csv

@@ -1,4 +1,4 @@
-,Unit,Values,Description
-start,--,"str or datetime-like; e.g. YYYY-MM-DD","Left bound of date range"
-end,--,"str or datetime-like; e.g. YYYY-MM-DD","Right bound of date range"
-inclusive,--,"One of {'neither', 'both', ‘left’, ‘right’}","Make the time interval closed to the ``left``, ``right``, or both sides ``both`` or neither side ``None``."
+,Unit,Values,Description
+start,--,str or datetime-like; e.g. YYYY-MM-DD,Left bound of date range
+end,--,str or datetime-like; e.g. YYYY-MM-DD,Right bound of date range
+inclusive,--,"One of {'neither', 'both', ‘left’, ‘right’}","Make the time interval closed to the ``left``, ``right``, or both sides ``both`` or neither side ``None``."

doc/configtables/solar.csv

@@ -2,13 +2,14 @@
 cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module can be ERA5 or SARAH-2.","Specifies the directory where the relevant weather data ist stored that is specified at ``atlite/cutouts`` configuration. Both ``sarah`` and ``era5`` work."
 resource,,,
 -- method,--,"Must be 'pv'","A superordinate technology type."
--- panel,--,"One of {'Csi', 'CdTe', 'KANENA'} as defined in `atlite <https://github.com/PyPSA/atlite/tree/master/atlite/resources/solarpanel>`__","Specifies the solar panel technology and its characteristic attributes."
+-- panel,--,"One of {'Csi', 'CdTe', 'KANENA'} as defined in `atlite <https://github.com/PyPSA/atlite/tree/master/atlite/resources/solarpanel>`_ . Can be a string or a dictionary with years as keys which denote the year another turbine model becomes available.","Specifies the solar panel technology and its characteristic attributes."
 -- orientation,,,
 -- -- slope,°,"Realistically any angle in [0., 90.]","Specifies the tilt angle (or slope) of the solar panel. A slope of zero corresponds to the face of the panel aiming directly overhead. A positive tilt angle steers the panel towards the equator."
 -- -- azimuth,°,"Any angle in [0., 360.]","Specifies the `azimuth <https://en.wikipedia.org/wiki/Azimuth>`_ orientation of the solar panel. South corresponds to 180.°."
 capacity_per_sqkm,:math:`MW/km^2`,float,"Allowable density of solar panel placement."
 correction_factor,--,float,"A correction factor for the capacity factor (availability) time series."
 corine,--,"Any subset of the `CORINE Land Cover code list <http://www.eea.europa.eu/data-and-maps/data/corine-land-cover-2006-raster-1/corine-land-cover-classes-and/clc_legend.csv/at_download/file>`_","Specifies areas according to CORINE Land Cover codes which are generally eligible for solar panel placement."
+luisa,--,"Any subset of the `LUISA Base Map codes in Annex 1 <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_","Specifies areas according to the LUISA Base Map codes which are generally eligible for solar panel placement."
 natura,bool,"{true, false}","Switch to exclude `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas. Area is excluded if ``true``."
 clip_p_max_pu,p.u.,float,"To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero."
 excluder_resolution,m,float,"Resolution on which to perform geographical elibility analysis."

doc/configtables/solving.csv

@@ -6,12 +6,19 @@ options,,,
 -- skip_iterations,bool,"{'true','false'}","Skip iterating, do not update impedances of branches. Defaults to true."
 -- rolling_horizon,bool,"{'true','false'}","Whether to optimize the network in a rolling horizon manner, where the snapshot range is split into slices of size `horizon` which are solved consecutively."
 -- seed,--,int,Random seed for increased deterministic behaviour.
+-- custom_extra_functionality,--,str,Path to a Python file with custom extra functionality code to be injected into the solving rules of the workflow relative to ``rules`` directory.
+-- io_api,string,"{'lp','mps','direct'}",Passed to linopy and determines the API used to communicate with the solver. With the ``'lp'`` and ``'mps'`` options linopy passes a file to the solver; with the ``'direct'`` option (only supported for HIGHS and Gurobi) linopy uses an in-memory python API resulting in better performance.
 -- track_iterations,bool,"{'true','false'}",Flag whether to store the intermediate branch capacities and objective function values are recorded for each iteration in ``network.lines['s_nom_opt_X']`` (where ``X`` labels the iteration)
 -- min_iterations,--,int,Minimum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run.
 -- max_iterations,--,int,Maximum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run.
 -- transmission_losses,int,[0-9],"Add piecewise linear approximation of transmission losses based on n tangents. Defaults to 0, which means losses are ignored."
 -- linearized_unit_commitment,bool,"{'true','false'}",Whether to optimise using the linearized unit commitment formulation.
 -- horizon,--,int,Number of snapshots to consider in each iteration. Defaults to 100.
+constraints ,,,
+-- CCL,bool,"{'true','false'}",Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``.
+-- EQ,bool/string,"{'false',`n(c| )``; i.e. ``0.5``-``0.7c``}",Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption.
+-- BAU,bool,"{'true','false'}",Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities``
+-- SAFE,bool,"{'true','false'}",Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network.
 solver,,,
 -- name,--,"One of {'gurobi', 'cplex', 'cbc', 'glpk', 'ipopt'}; potentially more possible",Solver to use for optimisation problems in the workflow; e.g. clustering and linear optimal power flow.
 -- options,--,Key listed under ``solver_options``.,Link to specific parameter settings.

doc/configtables/toplevel.csv

@@ -18,7 +18,7 @@ enable,,,
 -- build_natura_raster,bool,"{true, false}","Switch to enable the creation of the raster ``natura.tiff`` via the rule :mod:`build_natura_raster`."
 -- retrieve_natura_raster,bool,"{true, false}","Switch to enable the retrieval of ``natura.tiff`` from zenodo with :mod:`retrieve_natura_raster`."
 -- custom_busmap,bool,"{true, false}","Switch to enable the use of custom busmaps in rule :mod:`cluster_network`. If activated the rule looks for provided busmaps at ``data/custom_busmap_elec{weather_year}_s{simpl}_{clusters}.csv`` which should have the same format as ``resources/busmap_elec{weather_year}_s{simpl}_{clusters}.csv``, i.e. the index should contain the buses of ``networks/elec_s{simpl}.nc``."
--- drop_leap_days,bool,"{true, false}","Switch to drop February 29 from all time-dependent data in leap years"
+-- drop_leap_day,bool,"{true, false}","Switch to drop February 29 from all time-dependent data in leap years"
 =======
 co2_budget,--,"Dictionary with planning horizons as keys.","CO2 budget as a fraction of 1990 emissions. Overwritten if ``CO2Lx`` or ``cb`` are set in ``{sector_opts}`` wildcard"
 >>>>>>> master

doc/configuration.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 
@@ -9,7 +9,7 @@
 Configuration
 ##########################################
 
-PyPSA-Eur has several configuration options which are documented in this section and are collected in a ``config/config.yaml`` file located in the root directory. Users should copy the provided default configuration (``config/config.default.yaml``) and amend their own modifications and assumptions in the user-specific configuration file (``config/config.yaml``); confer installation instructions at :ref:`defaultconfig`.
+PyPSA-Eur has several configuration options which are documented in this section and are collected in a ``config/config.yaml`` file. This file defines deviations from the default configuration (``config/config.default.yaml``); confer installation instructions at :ref:`defaultconfig`.
 
 .. _toplevel_cf:
 
@@ -387,7 +387,7 @@ overwrite the existing values.
 
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
-   :start-after:   type:
+   :start-after: # docs-load
    :end-before: # docs
 
 .. csv-table::
@@ -565,6 +565,21 @@ The list of available biomass is given by the category in `ENSPRESO_BIOMASS <htt
    use ``min`` in ``p_nom_max:`` for more `
    conservative assumptions.
 
+.. _adjustments_cf:
+
+``adjustments``
+=============
+
+.. literalinclude:: ../config/config.default.yaml
+   :language: yaml
+   :start-at: adjustments:
+   :end-before: # docs
+
+.. csv-table::
+   :header-rows: 1
+   :widths: 22,7,22,33
+   :file: configtables/adjustments.csv
+
 .. _solving_cf:
 
 ``solving``

doc/contributing.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 

doc/costs.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 
@@ -9,7 +9,7 @@ Techno-Economic Assumptions
 
 The database of cost assumptions is retrieved from the repository
 `PyPSA/technology-data <https://github.com/pypsa/technology-data>`_ and then
-saved to a file ``data/costs_{year}.csv``. The ``config/config.yaml`` provides options
+saved to a file ``resources/costs_{year}.csv``. The ``config/config.yaml`` provides options
 to choose a reference year and use a specific version of the repository.
 
 .. literalinclude:: ../config/config.default.yaml
@@ -50,7 +50,7 @@ Modifying Assumptions
 Some cost assumptions (e.g. marginal cost and capital cost) can be directly
 set in the ``config/config.yaml`` (cf. Section  :ref:`costs_cf`  in
 :ref:`config`). To change cost assumptions in more detail, make a copy of
-``data/costs_{year}.csv`` and reference the new cost file in the ``Snakefile``:
+``resources/costs_{year}.csv`` and reference the new cost file in the ``Snakefile``:
 
 .. literalinclude:: ../Snakefile
    :start-at: COSTS

doc/foresight.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2021-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2021-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 
@@ -41,7 +41,7 @@ Perfect foresight scenarios
 
 .. warning::
 
-  Perfect foresight is currently implemented as a first test version.
+  Perfect foresight is currently implemented as an experimental test version.
 
 For running perfect foresight scenarios, you can adjust the
  ``config/config.perfect.yaml``:

doc/index.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 
@@ -35,6 +35,8 @@ PyPSA-Eur: A Sector-Coupled Open Optimisation Model of the European Energy Syste
    :target: https://stackoverflow.com/questions/tagged/pypsa
    :alt: Stackoverflow
 
+|
+
 PyPSA-Eur is an open model dataset of the European energy system at the
 transmission network level that covers the full ENTSO-E area. It covers demand
 and supply for all energy sectors. From version v0.8.0, PyPSA-Eur includes all
@@ -209,24 +211,6 @@ If you want to cite a specific PyPSA-Eur version, each release of PyPSA-Eur is s
    :target: https://doi.org/10.5281/zenodo.3520874
 
 
-Pre-Built Networks as a Dataset
-===============================
-
-There are pre-built networks available as a dataset on Zenodo as well for every release of PyPSA-Eur.
-
-.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3601881.svg
-   :target: https://doi.org/10.5281/zenodo.3601881
-
-The included ``.nc`` files are PyPSA network files which can be imported with PyPSA via:
-
-.. code:: python
-
-    import pypsa
-
-    filename = "elec_s_1024_ec.nc"  # example
-    n = pypsa.Network(filename)
-
-
 Operating Systems
 =================
 

doc/installation.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 
@@ -31,7 +31,7 @@ Install Python Dependencies
 
 PyPSA-Eur relies on a set of other Python packages to function.
 We recommend using the package manager `mamba <https://mamba.readthedocs.io/en/latest/>`_ to install them and manage your environments.
-For instructions for your operating system follow the ``mamba`` `installation guide <https://mamba.readthedocs.io/en/latest/installation.html>`_.
+For instructions for your operating system follow the ``mamba`` `installation guide <https://mamba.readthedocs.io/en/latest/installation/mamba-installation.html>`_.
 You can also use ``conda`` equivalently.
 
 The package requirements are curated in the `envs/environment.yaml <https://github.com/PyPSA/pypsa-eur/blob/master/envs/environment.yaml>`_ file.
@@ -79,31 +79,9 @@ Nevertheless, you can still use open-source solvers for smaller problems.
     `Instructions how to install a solver in the documentation of PyPSA <https://pypsa.readthedocs.io/en/latest/installation.html#getting-a-solver-for-linear-optimisation>`_
 
 .. note::
-    The rules :mod:`cluster_network` and :mod:`simplify_network` solve a quadratic optimisation problem for clustering.
-    The open-source solvers Cbc and GlPK cannot handle this. A fallback to Ipopt is implemented in this case, but requires
-    it to be installed. For an open-source solver setup install in your ``conda`` environment on OSX/Linux
-
-    .. code:: bash
-
-        mamba activate pypsa-eur
-        mamba install -c conda-forge ipopt coincbc
-
-    and on Windows
-
-    .. code:: bash
-
-        mamba activate pypsa-eur
-        mamba install -c conda-forge ipopt glpk
-
-    For HiGHS, run
-
-    .. code:: bash
-
-        mamba activate pypsa-eur
-        mamba install -c conda-forge ipopt
-        pip install highspy
-
-    For Gurobi, run
+    The rules :mod:`cluster_network` and :mod:`simplify_network` solve a mixed-integer quadratic optimisation problem for clustering.
+    The open-source solvers HiGHS, Cbc and GlPK cannot handle this. A fallback to SCIP is implemented in this case.
+    For an open-source solver setup install in your ``conda`` environment on OSX/Linux. To install the default solver Gurobi, run
 
     .. code:: bash
 
@@ -118,11 +96,10 @@ Nevertheless, you can still use open-source solvers for smaller problems.
 Handling Configuration Files
 ============================
 
-PyPSA-Eur has several configuration options that must be specified in a
-``config/config.yaml`` file located in the root directory. An example configuration
-``config/config.default.yaml`` is maintained in the repository, which will be used to
-automatically create your customisable ``config/config.yaml`` on first use. More
-details on the configuration options are in :ref:`config`.
+PyPSA-Eur has several configuration options that users can specify in a
+``config/config.yaml`` file. The default configuration
+``config/config.default.yaml`` is maintained in the repository. More details on
+the configuration options are in :ref:`config`.
 
 You can also use ``snakemake`` to specify another file, e.g.
 ``config/config.mymodifications.yaml``, to update the settings of the ``config/config.yaml``.
@@ -130,8 +107,3 @@ You can also use ``snakemake`` to specify another file, e.g.
 .. code:: bash
 
     .../pypsa-eur % snakemake -call --configfile config/config.mymodifications.yaml
-
-.. warning::
-    Users are advised to regularly check their own ``config/config.yaml`` against changes
-    in the ``config/config.default.yaml`` when pulling a new version from the remote
-    repository.

doc/introduction.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 
@@ -74,7 +74,7 @@ what data to retrieve and what files to produce. Details are explained in
 :ref:`wildcards` and :ref:`scenario`.
 
 The model also has several further configuration options collected in the
-``config/config.yaml`` file located in the root directory, which that are not part of
+``config/config.default.yaml`` file located in the root directory, which that are not part of
 the scenarios. Options are explained in :ref:`config`.
 
 Folder Structure
@@ -89,8 +89,8 @@ Folder Structure
 - ``results``: Stores the solved PyPSA network data, summary files and plots.
 - ``logs``: Stores log files.
 - ``benchmarks``: Stores ``snakemake`` benchmarks.
-- ``test``: Includes the test configuration files used for continuous integration.
 - ``doc``: Includes the documentation of PyPSA-Eur.
+- ``graphics``: Includes some graphics for the documentation of PyPSA-Eur.
 
 System Requirements
 ===================

doc/licenses.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2023-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 

doc/limitations.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 

doc/make.bat

@@ -1,4 +1,4 @@
-REM SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+REM SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors
 REM SPDX-License-Identifier: MIT
 
 @ECHO OFF

doc/plotting.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 
@@ -22,7 +22,22 @@ Rule ``plot_summary``
 
 .. _map_plot:
 
-Rule ``plot_network``
-========================
+Rule ``plot_power_network``
+===========================
 
-.. automodule:: plot_network
+.. automodule:: plot_power_network
+
+Rule ``plot_power_network_perfect``
+===================================
+
+.. automodule:: plot_power_network_perfect
+
+Rule ``plot_hydrogen_network``
+==============================
+
+.. automodule:: plot_hydrogen_network
+
+Rule ``plot_gas_network``
+=========================
+
+.. automodule:: plot_gas_network

doc/preparation.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 
@@ -94,6 +94,13 @@ Rule ``build_electricity_demand``
 
 .. automodule:: build_electricity_demand
 
+.. _monthlyprices:
+
+Rule ``build_monthly_prices``
+=============================
+
+.. automodule:: build_monthly_prices
+
 .. _ship:
 
 Rule ``build_ship_raster``
@@ -102,6 +109,12 @@ Rule ``build_ship_raster``
 
 .. automodule:: build_ship_raster
 
+.. _availabilitymatrixmdua:
+
+Rule ``determine_availability_matrix_MD_UA``
+============================================
+
+.. automodule:: determine_availability_matrix_MD_UA
 
 .. _renewableprofiles:
 

doc/publications.bib

@@ -1,5 +1,5 @@
 @Comment{
-SPDX-FileCopyrightText: 2023 The PyPSA-Eur Authors
+SPDX-FileCopyrightText: 2023-2024 The PyPSA-Eur Authors
 
 SPDX-License-Identifier: CC0-1.0
 }

doc/publications.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2023-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 

doc/release_notes.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 
@@ -9,58 +9,306 @@ Release Notes
 
 Upcoming Release
 ================
+* Upgrade default techno-economic assumptions to ``technology-data`` v0.8.1.
 
-* Pin ``snakemake`` version to below 8.0.0, as the new version is not yet
-  supported by ``pypsa-eur``.
+* Linearly interpolate missing investment periods in year-dependent
+  configuration options.
 
-* Updated Global Energy Monitor LNG terminal data to March 2023 version.
+* Added new scenario management that supports the simultaneous execution of
+  multiple scenarios with a single ``snakemake`` call. For this purpose, a
+  ``scenarios.yaml`` file is introduced which contains customizable scenario
+  names with configuration overrides. To enable it, set the ``run: scenarios:
+  true`` and define the list of scenario names to run under ``run: name:`` in
+  the configuration file. The latter must be a subset of toplevel keys in the
+  scenario file.
 
-* For industry distribution, use EPRTR as fallback if ETS data is not available.
+  - To get started, a scenarios template file ``config/scenarios.template.yaml``
+    is included in the repository, which is copied to ``config/scenarios.yaml``
+    on first use.
 
-* The minimum capacity for renewable generators when using the myopic option has been fixed.
+  - The scenario file can be changed via ``run: scenarios: file:``.
 
-* Files downloaded from zenodo are now write-protected to prevent accidental re-download.
+  - If scenario management is activated with ``run: scenarios: enable: true``, a
+    new wildcard ``{run}`` is introduced. This means that the configuration
+    settings may depend on the new ``{run}`` wildcard. Therefore, a new
+    ``config_provider()`` function is used in the ``Snakefile`` and ``.smk``
+    files, which takes wildcard values into account. The calls to the ``config``
+    object have been reduced in ``.smk`` files since there is no awareness of
+    wildcard values outside rule definitions.
 
-* Files extracted from sector-coupled data bundle have been moved from ``data/`` to ``data/sector-bundle``.
+  - The scenario files can also be programmatically created using the template
+    script ``config/create_scenarios.py``. This script can be run with
+    ``snakemake -j1 create_scenarios`` and creates the scenarios file referenced
+    under ``run: scenarios: file:``.
 
-* New feature multi-decade optimisation with  perfect foresight.
+  - The setting ``run: name: all`` will run all scenarios in
+    ``config/scenarios.yaml``. Otherwise, it will run those passed as list in
+    ``run: name:`` as long as ``run: scenarios: enable: true``.
 
-* It is now possible to specify years for biomass potentials which do not exist
-  in the JRC-ENSPRESO database, e.g. 2037. These are linearly interpolated.
+  - The setting ``run: shared_resources:`` indicates via a boolean whether the
+    resources should be encapsulated by the ``run: name:``. The special setting
+    ``run: shared_resources: base`` shares resources until ``add_electricity``
+    that do not contain wildcards other than ``{"technology", "year",
+    "scope"}``.
 
-* In pathway mode, the biomass potential is linked to the investment year.
+  - Added new configuration options for all ``{opts}`` and ``{sector_opts}``
+    wildcard values to create a unique configuration file (``config.yaml``) per
+    PyPSA network file. This is done with the help of a new function
+    ``update_config_from_wildcards()`` which parses configuration settings from
+    wildcards and updates the ``snakemake.config`` object. These updated
+    configuration settings are used in the scripts rather than directly parsed
+    values from ``snakemake.wildcards``.
 
-* Rule ``purge`` now initiates a dialog to confirm if purge is desired.
+  - The cost data was moved from ``data/costs_{year}.csv`` to
+    ``resources/costs_{year}.csv`` since it depends on configuration settings.
+    The ``retrieve_cost_data`` rule was changed to calling a Python script.
 
-* Rule ``retrieve_irena`` get updated values for renewables capacities.
+  - Moved time clustering settings to ``clustering: temporal:`` from
+    ``snapshots:`` so that the latter is only used to define the
+    ``pandas.DatetimeIndex`` which simplifies the scenario management.
 
-* Rule ``retrieve_wdpa`` updated to not only check for current and previous, but also potentially next months dataset availability.
+  - Collection rules get a new wildcard ``run=config["run"]["name"]`` so they
+    can collect outputs across different scenarios.
 
-* Split configuration to enable SMR and SMR CC.
+  - **Warning:** One caveat remains for the scenario management with myopic or
+    perfect foresight pathway optimisation. The first investment period must be
+    shared across all scenarios. The reason is that the ``wildcard_constraints``
+    defined for the rule ``add_existing_baseyear`` do not accept wildcard-aware
+    input functions (cf.
+    `https://github.com/snakemake/snakemake/issues/2703`_).
 
-* Bugfix: The unit of the capital cost of Haber-Bosch plants was corrected.
+* The outputs of the rule ``retrieve_gas_infrastructure_data`` no longer
+  marked as ``protected()`` as the download size is small.
 
-* The configuration setting for country focus weights when clustering the
-  network has been moved from ``focus_weights:`` to ``clustering:
-  focus_weights:``. Backwards compatibility to old config files is maintained.
+* Bugfix: allow modelling sector-coupled landlocked regions. (Fixed handling of offshore wind.)
 
-* Extend options for waste usage from Haber-Bosch, methanolisation and methanation.
+PyPSA-Eur 0.10.0 (19th February 2024)
+=====================================
 
-* Use electrolysis waste heat by default.
+**New Features**
 
-* Add new ``sector_opts`` wildcard option "nowasteheat" to disable all waste heat usage.
+* Improved representation of industry transition pathways. A new script was
+  added to interpolate industry sector ratios from today's status quo to future
+  systems (i.e. specific emissions and demands for energy and feedstocks). For
+  each country we gradually switch industry processes from today's specific
+  energy carrier usage per ton material output to the best-in-class energy
+  consumption of tomorrow. This is done on a per-country basis. The ratio of
+  today to tomorrow's energy consumption is set with the ``industry:
+  sector_ratios_fraction_future:`` parameter
+  (https://github.com/PyPSA/pypsa-eur/pull/929).
 
-* Set minimum part loads for PtX processes to 30% for methanolisation and methanation, and to 70% for Fischer-Tropsch synthesis.
+* Add new default to overdimension heating in individual buildings. This allows
+  them to cover heat demand peaks e.g. 10% higher than those in the data. The
+  disadvantage of manipulating the costs is that the capacity is then not quite
+  right. This way at least the costs are right
+  (https://github.com/PyPSA/pypsa-eur/pull/918).
 
-* Add VOM as marginal cost to PtX processes.
+* Allow industrial coal demand to be regional so its emissions can be included
+  in regional emission limits (https://github.com/PyPSA/pypsa-eur/pull/923).
 
-* Add pelletizing costs for biomass boilers.
+* Add option to specify to set a default heating lifetime for existing heating
+  (``existing_capacities: default_heating_lifetime:``)
+  (https://github.com/PyPSA/pypsa-eur/pull/918).
 
-* The ``mock_snakemake`` function can now be used with a Snakefile from a different directory using the new ``root_dir`` argument.
+* Added option to specify turbine and solar panel models for specific years as a
+  dictionary (e.g. ``renewable: onwind: resource: turbine:``). The years will be
+  interpreted as years from when the the corresponding turbine model substitutes
+  the previous model for new installations. This will only have an effect on
+  workflows with foresight ``"myopic"`` and still needs to be added foresight
+  option ``"perfect"`` (https://github.com/PyPSA/pypsa-eur/pull/912).
 
-* Switch to using hydrogen and electricity inputs for Haber-Bosch from https://github.com/PyPSA/technology-data.
+* New configuration option ``everywhere_powerplants`` to build conventional
+  powerplants everywhere, irrespective of existing powerplants locations, in the
+  network (https://github.com/PyPSA/pypsa-eur/pull/850).
 
-* Add option to capture CO2 contained in biogas when upgrading (``sector: biogas_to_gas_cc``).
+* Add the option to customise map projection in plotting config under
+  ``plotting: projection: name`` (https://github.com/PyPSA/pypsa-eur/pull/898).
+
+* Add support for the linopy ``io_api`` option under ``solving: options:
+  io_api:``. Set to ``"direct"`` to increase model reading and writing
+  performance for the highs and gurobi solvers on slow file systems
+  (https://github.com/PyPSA/pypsa-eur/pull/892).
+
+* It is now possible to determine the directory for shared resources by setting
+  `shared_resources` to a string (https://github.com/PyPSA/pypsa-eur/pull/906).
+
+* Improve ``mock_snakemake()`` for usage in Snakemake modules
+  (https://github.com/PyPSA/pypsa-eur/pull/869).
+
+**Breaking Changes**
+
+* Remove long-deprecated function ``attach_extendable_generators`` in
+  :mod:`add_electricity`.
+
+* Remove option for wave energy as technology data is not maintained.
+
+* The order of buses (bus0, bus1, ...) for DAC components has changed to meet
+  the convention of the other components. Therefore, `bus0` refers to the
+  electricity bus (input), `bus1` to the heat bus (input), 'bus2' to the CO2
+  atmosphere bus (input), and `bus3` to the CO2 storage bus (output)
+  (https://github.com/PyPSA/pypsa-eur/pull/901).
+
+**Changes**
+
+* Upgrade default techno-economic assumptions to ``technology-data`` v0.8.0.
+
+* Update hydrogen pipeline losses to latest data from Danish Energy Agency
+  (https://github.com/PyPSA/pypsa-eur/pull/933).
+
+* Move building of daily heat profile to its own rule
+  :mod:`build_hourly_heat_demand` from :mod:`prepare_sector_network`
+  (https://github.com/PyPSA/pypsa-eur/pull/884).
+
+* In :mod:`build_energy_totals`, district heating shares are now reported in a
+  separate file (https://github.com/PyPSA/pypsa-eur/pull/884).
+
+* Move calculation of district heating share to its own rule
+  :mod:`build_district_heat_share`
+  (https://github.com/PyPSA/pypsa-eur/pull/884).
+
+* Move building of distribution of existing heating to own rule
+  :mod:`build_existing_heating_distribution`. This makes the distribution of
+  existing heating to urban/rural, residential/services and spatially more
+  transparent (https://github.com/PyPSA/pypsa-eur/pull/884).
+
+* Default settings for recycling rates and primary product shares of high-value
+  chemicals have been set in accordance with the values used in `Neumann et al.
+  (2023) <https://doi.org/10.1016/j.joule.2023.06.016>`_ linearly interpolated
+  between 2020 and 2050. The recycling rates are based on data from `Agora
+  Energiewende (2021)
+  <https://static.agora-energiewende.de/fileadmin/Projekte/2021/2021_02_EU_CEAP/A-EW_254_Mobilising-circular-economy_study_WEB.pdf>`_.
+
+* Air-sourced heat pumps can now also be built in rural areas. Previously, only
+  ground-sourced heat pumps were considered for this category
+  (https://github.com/PyPSA/pypsa-eur/pull/890).
+
+* The default configuration ``config/config.default.yaml`` is now automatically
+  used as a base configuration file. The file ``config/config.yaml`` can now be
+  used to only define deviations from the default configuration. The
+  ``config/config.default.yaml`` is still copied into ``config/config.yaml`` on
+  first usage (https://github.com/PyPSA/pypsa-eur/pull/925).
+
+* Regions are assigned to all buses with unique coordinates in the network with
+  a preference given to substations. Previously, only substations had assigned
+  regions, but this could lead to issues when a high spatial resolution was
+  applied (https://github.com/PyPSA/pypsa-eur/pull/922).
+
+* Define global constraint for CO2 emissions on the final state of charge of the
+  CO2 atmosphere store. This gives a more sparse constraint that should improve
+  the performance of the solving process
+  (https://github.com/PyPSA/pypsa-eur/pull/862).
+
+* Switched the energy totals year from 2011 to 2013 to comply with the assumed
+  default weather year (https://github.com/PyPSA/pypsa-eur/pull/934).
+
+* Cluster residential and services heat buses by default. Can be disabled with
+  ``cluster_heat_buses: false`` (https://github.com/PyPSA/pypsa-eur/pull/877).
+
+* The rule ``plot_network`` has been split into separate rules for plotting
+  electricity, hydrogen and gas networks
+  (https://github.com/PyPSA/pypsa-eur/pull/900).
+
+* To determine the optimal topology to meet the number of clusters, the workflow
+  used pyomo in combination with ``ipopt`` or ``gurobi``. This dependency has
+  been replaced by using ``linopy`` in combination with ``scipopt`` or
+  ``gurobi``. The environment file has been updated accordingly
+  (https://github.com/PyPSA/pypsa-eur/pull/903).
+
+* The ``highs`` solver was added to the default environment file.
+
+* New default solver settings for COPT solver
+  (https://github.com/PyPSA/pypsa-eur/pull/882).
+
+* Data retrieval rules now use their own minimal conda environment. This can
+  avoid unnecessary reruns of the workflow
+  (https://github.com/PyPSA/pypsa-eur/pull/888).
+
+* Merged two OPSD time series data versions into such that the option ``load:
+  power_statistics:`` becomes superfluous and was hence removed
+  (https://github.com/PyPSA/pypsa-eur/pull/924).
+
+* The filtering of power plants in the ``config.default.yaml`` has been updated
+  regarding phased-out power plants in 2023.
+
+* Include all countries in ammonia production resource. This is so that the full
+  EU28 ammonia demand can be correctly subtracted in the rule
+  :mod:`build_industry_sector_ratios`
+  (https://github.com/PyPSA/pypsa-eur/pull/931).
+
+* Correctly source the existing heating technologies for buildings since the
+  source URL has changed. It represents the year 2012 and is only for buildings,
+  not district heating (https://github.com/PyPSA/pypsa-eur/pull/918).
+
+* Add warning when BEV availability weekly profile has negative values in
+  `build_transport_demand` (https://github.com/PyPSA/pypsa-eur/pull/858).
+
+* Time series clipping for very small values was added for Links
+  (https://github.com/PyPSA/pypsa-eur/pull/870).
+
+* A ``test.sh`` script was added to the repository to run the tests locally.
+
+* The CI now tests additionally against ``master`` versions of PyPSA, atlite and
+  powerplantmatching (https://github.com/PyPSA/pypsa-eur/pull/904).
+
+* A function ``sanitize_locations()`` was added to improve the coverage of the
+  ``location`` attribute of network components.
+
+**Bugs and Compatibility**
+
+* Bugfix: Do not reduce district heat share when building population-weighted
+  energy statistics. Previously the district heating share was being multiplied
+  by the population weighting, reducing the DH share with multiple nodes
+  (https://github.com/PyPSA/pypsa-eur/pull/884).
+
+* Bugfix: The industry coal emissions for industry were not properly tracked
+  (https://github.com/PyPSA/pypsa-eur/pull/923).
+
+* Bugfix: Correct units of subtracted chlorine and methanol demand in
+  :mod:`build_industry_sector_ratios`
+  (https://github.com/PyPSA/pypsa-eur/pull/930).
+
+* Various minor bugfixes to the perfect foresight workflow, though perfect
+  foresight must still be considered experimental
+  (https://github.com/PyPSA/pypsa-eur/pull/910).
+
+* Fix plotting of retrofitted hydrogen pipelines with myopic pathway
+  optimisation (https://github.com/PyPSA/pypsa-eur/pull/937).
+
+* Bugfix: Correct technology keys for the electricity production plotting to
+  work out the box.
+
+* Bugfix: Assure entering of code block which corrects Norwegian heat demand
+  (https://github.com/PyPSA/pypsa-eur/pull/870).
+
+* Stacktrace of uncaught exceptions should now be correctly included inside log
+  files (via `configure_logging(..)`)
+  (https://github.com/PyPSA/pypsa-eur/pull/875).
+
+* Bugfix: Correctly read out number of solver threads from configuration file
+  (https://github.com/PyPSA/pypsa-eur/pull/889).
+
+* Made copying default config file compatible with snakemake module
+  (https://github.com/PyPSA/pypsa-eur/pull/894).
+
+* Compatibility with ``pandas=2.2``
+  (https://github.com/PyPSA/pypsa-eur/pull/861).
+
+Special thanks for this release to Koen van Greevenbroek (`@koen-vg
+<https://github.com/koen-vg>`_) for various new features, bugfixes and taking
+care of deprecations.
+
+
+PyPSA-Eur 0.9.0 (5th January 2024)
+==================================
+
+**New Features**
+
+* Add option to specify losses for bidirectional links, e.g. pipelines or HVDC
+  links, in configuration file under ``sector: transmission_efficiency:``. Users
+  can specify static or length-dependent values as well as a length-dependent
+  electricity demand for compression, which is implemented as a multi-link to
+  the local electricity buses. The bidirectional links will then be split into
+  two unidirectional links with linked capacities (https://github.com/PyPSA/pypsa-eur/pull/739).
 
 * Merged option to extend geographical scope to Ukraine and Moldova. These
   countries are excluded by default and is currently constrained to power-sector
@@ -70,25 +318,264 @@ Upcoming Release
   Moldova). Moldova can currently only be included in conjunction with Ukraine
   due to the absence of demand data. The Crimean power system is manually
   reconnected to the main Ukrainian grid with the configuration option
-  `reconnect_crimea`.
+  `reconnect_crimea` (https://github.com/PyPSA/pypsa-eur/pull/321).
 
-* Validate downloads from Zenodo using MD5 checksums. This identifies corrupted
-  or incomplete downloads.
+* New experimental support for multi-decade optimisation with perfect foresight
+  (``foresight: perfect``). Maximum growth rates for carriers, global carbon
+  budget constraints and emission constraints for particular investment periods.
+
+* Add option to reference an additional source file where users can specify
+  custom ``extra_functionality`` constraints in the configuration file. The
+  default setting points to an empty hull at
+  ``data/custom_extra_functionality.py`` (https://github.com/PyPSA/pypsa-eur/pull/824).
 
 * Add locations, capacities and costs of existing gas storage using Global
   Energy Monitor's `Europe Gas Tracker
-  <https://globalenergymonitor.org/projects/europe-gas-tracker>`_.
+  <https://globalenergymonitor.org/projects/europe-gas-tracker>`_
+  (https://github.com/PyPSA/pypsa-eur/pull/835).
 
-* Remove HELMETH option.
+* Add option to use `LUISA Base Map
+  <https://publications.jrc.ec.europa.eu/repository/handle/JRC124621>`_ 50m land
+  coverage dataset for land eligibility analysis in
+  :mod:`build_renewable_profiles`. Settings are analogous to the CORINE dataset
+  but with the key ``luisa:`` in the configuration file. To leverage the
+  dataset's full advantages, set the excluder resolution to 50m
+  (``excluder_resolution: 50``). For land category codes, see `Annex 1 of the
+  technical documentation
+  <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_
+  (https://github.com/PyPSA/pypsa-eur/pull/842).
+
+* Add option to capture CO2 contained in biogas when upgrading (``sector:
+  biogas_to_gas_cc``) (https://github.com/PyPSA/pypsa-eur/pull/615).
+
+* If load shedding is activated, it is now applied to all carriers, not only
+  electricity (https://github.com/PyPSA/pypsa-eur/pull/784).
+
+* Add option for heat vents in district heating (``sector:
+  central_heat_vent:``). The combination of must-run conditions for some
+  power-to-X processes, waste heat usage enabled and decreasing heating demand,
+  can lead to infeasibilities in pathway optimisation for some investment
+  periods since larger Fischer-Tropsch capacities are needed in early years but
+  the waste heat exceeds the heat demand in later investment periods.
+  (https://github.com/PyPSA/pypsa-eur/pull/791).
+
+* Allow possibility to go from copperplated to regionally resolved methanol and
+  oil demand with switches ``sector: regional_methanol_demand: true`` and
+  ``sector: regional_oil_demand: true``. This allows nodal/regional CO2
+  constraints to be applied (https://github.com/PyPSA/pypsa-eur/pull/827).
+
+* Allow retrofitting of existing gas boilers to hydrogen boilers in pathway
+  optimisation.
+
+* Add option to add time-varying CO2 emission prices (electricity-only, ``costs:
+  emission_prices: co2_monthly_prices: true``). This is linked to the new
+  ``{opts}`` wildcard option ``Ept``.
+
+* Network clustering can now consider efficiency classes when aggregating
+  carriers. The option ``clustering: consider_efficiency_classes:`` aggregates
+  each carriers into the top 10-quantile (high), the bottom 90-quantile (low),
+  and everything in between (medium).
+
+* Added option ``conventional: dynamic_fuel_price:`` to consider the monthly
+  fluctuating fuel prices for conventional generators. Refer to the CSV file
+  ``data/validation/monthly_fuel_price.csv``.
+
+* For hydro-electricity, add switches ``flatten_dispatch`` to consider an upper
+  limit for the hydro dispatch. The limit is given by the average capacity
+  factor plus the buffer given in  ``flatten_dispatch_buffer``.
+
+* Extend options for waste heat usage from Haber-Bosch, methanolisation and
+  methanation (https://github.com/PyPSA/pypsa-eur/pull/834).
+
+* Add new ``sector_opts`` wildcard option "nowasteheat" to disable all waste
+  heat usage (https://github.com/PyPSA/pypsa-eur/pull/834).
+
+* Add new rule ``retrieve_irena`` to automatically retrieve up-to-date values
+  for existing renewables capacities (https://github.com/PyPSA/pypsa-eur/pull/756).
 
 * Print Irreducible Infeasible Subset (IIS) if model is infeasible. Only for
-  solvers with IIS support.
+  solvers with IIS support (https://github.com/PyPSA/pypsa-eur/pull/841).
+
+* More wildcard options now have a corresponding config entry. If the wildcard
+  is given, then its value is used. If the wildcard is not given but the options
+  in config are enabled, then the value from config is used. If neither is
+  given, the options are skipped (https://github.com/PyPSA/pypsa-eur/pull/827).
+
+* Validate downloads from Zenodo using MD5 checksums. This identifies corrupted
+  or incomplete downloads (https://github.com/PyPSA/pypsa-eur/pull/821).
+
+* Add rule ``sync`` to synchronise with a remote machine using the ``rsync``
+  library. Configuration settings are found under ``remote:``.
+
+**Breaking Changes**
+
+* Remove all negative loads on the ``co2 atmosphere`` bus representing emissions
+  for e.g. fixed fossil demands for transport oil. Instead these are handled
+  more transparently with a fixed transport oil demand and a link taking care of
+  the emissions to the ``co2 atmosphere`` bus. This is also a preparation for
+  endogenous transport optimisation, where demand will be subject to
+  optimisation (e.g. fuel switching in the transport sector)
+  (https://github.com/PyPSA/pypsa-eur/pull/827).
+
+* Process emissions from steam crackers (i.e. naphtha processing for HVC) are
+  now piped from the consumption link to the process emissions bus where the
+  model can decide about carbon capture. Previously the process emissions for
+  naphtha were a fixed load (https://github.com/PyPSA/pypsa-eur/pull/827).
+
+* Distinguish between stored and sequestered CO2. Stored CO2 is stored
+  overground in tanks and can be used for CCU (e.g. methanolisation).
+  Sequestered CO2 is stored underground and can no longer be used for CCU. This
+  distinction is made because storage in tanks is more expensive than
+  underground storage. The link that connects stored and sequestered CO2 is
+  unidirectional (https://github.com/PyPSA/pypsa-eur/pull/844).
+
+* Files extracted from sector-coupled data bundle have been moved from ``data/``
+  to ``data/sector-bundle``.
+
+* Split configuration to enable SMR and SMR CC (``sector: smr:`` and ``sector:
+  smr_cc:``) (https://github.com/PyPSA/pypsa-eur/pull/757).
+
+* Add separate option to add resistive heaters to the technology choices
+  (``sector: resistive_heaters:``). Previously they were always added when
+  boilers were added (https://github.com/PyPSA/pypsa-eur/pull/808).
+
+* Remove HELMETH option (``sector: helmeth:``).
+
+* Remove "conservative" renewable potentials estimation option
+  (https://github.com/PyPSA/pypsa-eur/pull/838).
+
+* With this release we stop posting updates to the network pre-builts.
+
+**Changes**
+
+* Updated Global Energy Monitor LNG terminal data to March 2023 version
+  (https://github.com/PyPSA/pypsa-eur/pull/707).
+
+* For industry distribution, use EPRTR as fallback if ETS data is not available
+  (https://github.com/PyPSA/pypsa-eur/pull/721).
+
+* It is now possible to specify years for biomass potentials which do not exist
+  in the JRC-ENSPRESO database, e.g. 2037. These are linearly interpolated
+  (https://github.com/PyPSA/pypsa-eur/pull/744).
+
+* In pathway mode, the biomass potential is linked to the investment year
+  (https://github.com/PyPSA/pypsa-eur/pull/744).
+
+* Increase allowed deployment density of solar to 5.1 MW/sqkm by default.
+
+* Default to full electrification of land transport by 2050.
+
+* Provide exogenous transition settings in 5-year steps.
+
+* Default to approximating transmission losses in HVAC lines
+  (``transmission_losses: 2``).
+
+* Use electrolysis waste heat by default.
+
+* Set minimum part loads for PtX processes to 30% for methanolisation and
+  methanation, and to 70% for Fischer-Tropsch synthesis.
+
+* Add VOM as marginal cost to PtX processes
+  (https://github.com/PyPSA/pypsa-eur/pull/830).
+
+* Add pelletizing costs for biomass boilers (https://github.com/PyPSA/pypsa-eur/pull/833).
+
+* Update default offshore wind turbine model to "NREL Reference 2020 ATB 5.5 MW"
+  (https://github.com/PyPSA/pypsa-eur/pull/832).
+
+* Switch to using hydrogen and electricity inputs for Haber-Bosch from
+  https://github.com/PyPSA/technology-data (https://github.com/PyPSA/pypsa-eur/pull/831).
+
+* The configuration setting for country focus weights when clustering the
+  network has been moved from ``focus_weights:`` to ``clustering:
+  focus_weights:``. Backwards compatibility to old config files is maintained
+  (https://github.com/PyPSA/pypsa-eur/pull/794).
+
+* The ``mock_snakemake`` function can now be used with a Snakefile from a
+  different directory using the new ``root_dir`` argument
+  (https://github.com/PyPSA/pypsa-eur/pull/771).
+
+* Rule ``purge`` now initiates a dialog to confirm if purge is desired
+  (https://github.com/PyPSA/pypsa-eur/pull/745).
+
+* Files downloaded from zenodo are now write-protected to prevent accidental
+  re-download (https://github.com/PyPSA/pypsa-eur/pull/730).
+
+* Performance improvements for rule ``build_ship_raster``
+  (https://github.com/PyPSA/pypsa-eur/pull/845).
+
+* Improve time logging in :mod:`build_renewable_profiles`
+  (https://github.com/PyPSA/pypsa-eur/pull/837).
+
+* In myopic pathway optimisation, disable power grid expansion if line volume
+  already hit (https://github.com/PyPSA/pypsa-eur/pull/840).
+
+* JRC-ENSPRESO data is now downloaded from a Zenodo mirror because the link was
+  unreliable (https://github.com/PyPSA/pypsa-eur/pull/801).
+
+* Add focus weights option for clustering to documentation
+  (https://github.com/PyPSA/pypsa-eur/pull/781).
+
+* Add proxy for biomass transport costs if no explicit biomass transport network
+  is considered (https://github.com/PyPSA/pypsa-eur/pull/711).
 
 **Bugs and Compatibility**
 
-* A bug preventing custom powerplants specified in ``data/custom_powerplants.csv`` was fixed. (https://github.com/PyPSA/pypsa-eur/pull/732)
-* Fix nodal fraction in ``add_existing_year`` when using distributed generators
-* Fix typo in buses definition for oil boilers in ``add_industry`` in ``prepare_sector_network``
+* The minimum PyPSA version is now 0.26.1.
+
+* Update to ``tsam>=0.2.3`` for performance improvents in temporal clustering.
+
+* Pin ``snakemake`` version to below 8.0.0, as the new version is not yet
+  supported. The next release will switch to the requirement ``snakemake>=8``.
+
+* Bugfix: Add coke and coal demand for integrated steelworks
+  (https://github.com/PyPSA/pypsa-eur/pull/718).
+
+* Bugfix: Make :mod:`build_renewable_profiles` consider subsets of cutout time
+  scope (https://github.com/PyPSA/pypsa-eur/pull/709).
+
+* Bugfix: In :mod:`simplify network`, remove 'underground' column to avoid
+  consense error (https://github.com/PyPSA/pypsa-eur/pull/714).
+
+* Bugfix: Fix in :mod:`add_existing_baseyear` to account for the case when there
+  is no rural heating demand for some nodes in network
+  (https://github.com/PyPSA/pypsa-eur/pull/706).
+
+* Bugfix: The unit of the capital cost of Haber-Bosch plants was corrected
+  (https://github.com/PyPSA/pypsa-eur/pull/829).
+
+* The minimum capacity for renewable generators when using the myopic option has
+  been fixed (https://github.com/PyPSA/pypsa-eur/pull/728).
+
+* Compatibility for running with single node and single country
+  (https://github.com/PyPSA/pypsa-eur/pull/839).
+
+* A bug preventing the addition of custom powerplants specified in
+  ``data/custom_powerplants.csv`` was fixed.
+  (https://github.com/PyPSA/pypsa-eur/pull/732)
+
+* Fix nodal fraction in :mod:`add_existing_year` when using distributed
+  generators (https://github.com/PyPSA/pypsa-eur/pull/798).
+
+* Bugfix: District heating without progress caused division by zero
+  (https://github.com/PyPSA/pypsa-eur/pull/796).
+
+* Bugfix: Drop duplicates in :mod:`build_industrial_distribution_keys`, which
+  can occur through the geopandas ``.sjoin()`` function if a point is located on
+  a border (https://github.com/PyPSA/pypsa-eur/pull/726).
+
+* For network clustering fall back to ``ipopt`` when ``highs`` is designated
+  solver (https://github.com/PyPSA/pypsa-eur/pull/795).
+
+* Fix typo in buses definition for oil boilers in ``add_industry`` in
+  :mod:`prepare_sector_network` (https://github.com/PyPSA/pypsa-eur/pull/812).
+
+* Resolve code issues for endogenous building retrofitting. Select correct
+  sector names, address deprecations, distinguish between district heating,
+  decentral heating in urban areas or rural areas for floor area calculations
+  (https://github.com/PyPSA/pypsa-eur/pull/808).
+
+* Addressed various deprecations.
 
 
 PyPSA-Eur 0.8.1 (27th July 2023)
@@ -1477,8 +1964,4 @@ Release Process
 
 * Make a `GitHub release <https://github.com/PyPSA/pypsa-eur-sec/releases>`_, which automatically triggers archiving to the `zenodo code repository <https://doi.org/10.5281/zenodo.3520874>`_ with `MIT license <https://opensource.org/licenses/MIT>`_.
 
-* Create pre-built networks for ``config.default.yaml`` by running ``snakemake -call prepare_sector_networks``.
-
-* Upload pre-built networks to `zenodo data repository <https://doi.org/10.5281/zenodo.3601881>`_ with `CC BY 4.0 <https://creativecommons.org/licenses/by/4.0/>`_ license.
-
 * Send announcement on the `PyPSA mailing list <https://groups.google.com/forum/#!forum/pypsa>`_.

doc/requirements.txt

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2019-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2019-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 

doc/retrieve.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 
@@ -91,7 +91,7 @@ None.
 
 **Outputs**
 
-- ``resources/load_raw.csv``
+- ``data/electricity_demand_raw.csv``
 
 
 Rule ``retrieve_cost_data``

doc/sector.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2023-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 
@@ -20,6 +20,12 @@ Rule ``add_existing_baseyear``
 
 .. automodule:: add_existing_baseyear
 
+Rule ``build_existing_heating_distribution``
+==============================================================================
+
+.. automodule:: build_existing_heating_distribution
+
+
 Rule ``build_ammonia_production``
 ==============================================================================
 
@@ -60,10 +66,20 @@ Rule ``build_gas_network``
 
 .. automodule:: build_gas_network
 
-Rule ``build_heat_demand``
+Rule ``build_daily_heat_demand``
 ==============================================================================
 
-.. automodule:: build_heat_demand
+.. automodule:: build_daily_heat_demand
+
+Rule ``build_hourly_heat_demand``
+==============================================================================
+
+.. automodule:: build_hourly_heat_demand
+
+Rule ``build_district_heat_share``
+==============================================================================
+
+.. automodule:: build_district_heat_share
 
 Rule ``build_industrial_distribution_key``
 ==============================================================================

doc/simplification.rst

@@ -1,7 +1,7 @@
 
 
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 

doc/solving.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 

doc/spatial_resolution.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2021-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2021-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 
@@ -45,7 +45,7 @@ Here are some examples of how spatial resolution is set for different sectors in
 
 •	CO2: It can be modeled as a single node for Europe or it can be nodally resolved with CO2 transport pipelines if activated in the `config <https://github.com/PyPSA/pypsa-eur-sec/blob/3daff49c9999ba7ca7534df4e587e1d516044fc3/config.default.yaml#L248>`_. It should mentioned that in single node mode a transport and storage cost is added for sequestered CO2, the cost of which can be adjusted in the `config <https://github.com/PyPSA/pypsa-eur-sec/blob/3daff49c9999ba7ca7534df4e587e1d516044fc3/config.default.yaml#L247>`_.
 
-•	Liquid hydrocarbons: Modeled as a single node for Europe, since transport costs for liquids are low and no bottlenecks are expected.
+•	Carbonaceous fuels: Modeled as a single node for Europe by default, since transport costs for liquids are low and no bottlenecks are expected. Can be regionally resolved in configuration.
 
 **Electricity distribution network**
 

doc/supply_demand.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2021-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2021-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 

doc/support.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 

doc/tutorial.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 
@@ -145,82 +145,89 @@ This triggers a workflow of multiple preceding jobs that depend on each rule's i
         graph[bgcolor=white, margin=0];
         node[shape=box, style=rounded, fontname=sans,                 fontsize=10, penwidth=2];
         edge[penwidth=2, color=grey];
-            0[label = "solve_network", color = "0.33 0.6 0.85", style="rounded"];
-            1[label = "prepare_network\nll: copt\nopts: Co2L-24H", color = "0.03 0.6 0.85", style="rounded"];
-            2[label = "add_extra_components", color = "0.45 0.6 0.85", style="rounded"];
-            3[label = "cluster_network\nclusters: 6", color = "0.46 0.6 0.85", style="rounded"];
-            4[label = "simplify_network\nsimpl: ", color = "0.52 0.6 0.85", style="rounded"];
-            5[label = "add_electricity", color = "0.55 0.6 0.85", style="rounded"];
-            6[label = "build_renewable_profiles\ntechnology: solar", color = "0.15 0.6 0.85", style="rounded"];
-            7[label = "base_network", color = "0.37 0.6 0.85", style="rounded,dashed"];
-            8[label = "build_shapes", color = "0.07 0.6 0.85", style="rounded,dashed"];
-            9[label = "retrieve_databundle", color = "0.60 0.6 0.85", style="rounded"];
-            10[label = "retrieve_natura_raster", color = "0.42 0.6 0.85", style="rounded"];
-            11[label = "build_bus_regions", color = "0.09 0.6 0.85", style="rounded,dashed"];
-            12[label = "build_renewable_profiles\ntechnology: onwind", color = "0.15 0.6 0.85", style="rounded"];
-            13[label = "build_renewable_profiles\ntechnology: offwind-ac", color = "0.15 0.6 0.85", style="rounded"];
-            14[label = "build_ship_raster", color = "0.02 0.6 0.85", style="rounded"];
-            15[label = "retrieve_ship_raster", color = "0.40 0.6 0.85", style="rounded"];
-            16[label = "build_renewable_profiles\ntechnology: offwind-dc", color = "0.15 0.6 0.85", style="rounded"];
-            17[label = "build_line_rating", color = "0.32 0.6 0.85", style="rounded"];
-            18[label = "retrieve_cost_data\nyear: 2030", color = "0.50 0.6 0.85", style="rounded"];
-            19[label = "build_powerplants", color = "0.64 0.6 0.85", style="rounded,dashed"];
-            20[label = "build_electricity_demand", color = "0.13 0.6 0.85", style="rounded,dashed"];
-            21[label = "retrieve_electricity_demand", color = "0.31 0.6 0.85", style="rounded"];
-            22[label = "copy_config", color = "0.23 0.6 0.85", style="rounded"];
+            0[label = "solve_network", color = "0.39 0.6 0.85", style="rounded"];
+            1[label = "prepare_network\nll: copt\nopts: Co2L-24H", color = "0.29 0.6 0.85", style="rounded"];
+            2[label = "add_extra_components", color = "0.28 0.6 0.85", style="rounded"];
+            3[label = "cluster_network\nclusters: 6", color = "0.19 0.6 0.85", style="rounded"];
+            4[label = "simplify_network\nsimpl: ", color = "0.01 0.6 0.85", style="rounded"];
+            5[label = "add_electricity", color = "0.49 0.6 0.85", style="rounded"];
+            6[label = "build_renewable_profiles\ntechnology: solar", color = "0.21 0.6 0.85", style="rounded"];
+            7[label = "base_network", color = "0.27 0.6 0.85", style="rounded"];
+            8[label = "build_shapes", color = "0.26 0.6 0.85", style="rounded"];
+            9[label = "retrieve_databundle", color = "0.59 0.6 0.85", style="rounded"];
+            10[label = "retrieve_natura_raster", color = "0.47 0.6 0.85", style="rounded"];
+            11[label = "build_bus_regions", color = "0.13 0.6 0.85", style="rounded"];
+            12[label = "retrieve_cutout\ncutout: be-03-2013-era5", color = "0.36 0.6 0.85", style="rounded,dashed"];
+            13[label = "build_renewable_profiles\ntechnology: onwind", color = "0.21 0.6 0.85", style="rounded"];
+            14[label = "build_renewable_profiles\ntechnology: offwind-ac", color = "0.21 0.6 0.85", style="rounded"];
+            15[label = "build_ship_raster", color = "0.00 0.6 0.85", style="rounded"];
+            16[label = "retrieve_ship_raster", color = "0.51 0.6 0.85", style="rounded,dashed"];
+            17[label = "build_renewable_profiles\ntechnology: offwind-dc", color = "0.21 0.6 0.85", style="rounded"];
+            18[label = "build_line_rating", color = "0.05 0.6 0.85", style="rounded"];
+            19[label = "retrieve_cost_data\nyear: 2030", color = "0.15 0.6 0.85", style="rounded"];
+            20[label = "build_powerplants", color = "0.54 0.6 0.85", style="rounded"];
+            21[label = "build_electricity_demand", color = "0.52 0.6 0.85", style="rounded"];
+            22[label = "retrieve_electricity_demand", color = "0.22 0.6 0.85", style="rounded"];
+            23[label = "copy_config", color = "0.44 0.6 0.85", style="rounded"];
             1 -> 0
-            22 -> 0
+            23 -> 0
             2 -> 1
-            18 -> 1
+            19 -> 1
             3 -> 2
-            18 -> 2
+            19 -> 2
             4 -> 3
-            18 -> 3
+            19 -> 3
             5 -> 4
-            18 -> 4
+            19 -> 4
             11 -> 4
             6 -> 5
-            12 -> 5
             13 -> 5
-            16 -> 5
-            7 -> 5
+            14 -> 5
             17 -> 5
+            7 -> 5
             18 -> 5
-            11 -> 5
             19 -> 5
-            9 -> 5
+            11 -> 5
             20 -> 5
+            9 -> 5
+            21 -> 5
             8 -> 5
             7 -> 6
             9 -> 6
             10 -> 6
             8 -> 6
             11 -> 6
+            12 -> 6
             8 -> 7
             9 -> 8
             8 -> 11
             7 -> 11
-            7 -> 12
-            9 -> 12
-            10 -> 12
-            8 -> 12
-            11 -> 12
             7 -> 13
             9 -> 13
             10 -> 13
-            14 -> 13
             8 -> 13
             11 -> 13
+            12 -> 13
+            7 -> 14
+            9 -> 14
+            10 -> 14
             15 -> 14
-            7 -> 16
-            9 -> 16
-            10 -> 16
-            14 -> 16
-            8 -> 16
-            11 -> 16
+            8 -> 14
+            11 -> 14
+            12 -> 14
+            16 -> 15
+            12 -> 15
             7 -> 17
-            7 -> 19
-            21 -> 20
+            9 -> 17
+            10 -> 17
+            15 -> 17
+            8 -> 17
+            11 -> 17
+            12 -> 17
+            7 -> 18
+            12 -> 18
+            7 -> 20
+            22 -> 21
     }
 
 |
@@ -230,26 +237,29 @@ In the terminal, this will show up as a list of jobs to be run:
 .. code:: bash
 
     Building DAG of jobs...
-    job                         count    min threads    max threads
-    ------------------------  -------  -------------  -------------
-    add_electricity                 1              1              1
-    add_extra_components            1              1              1
-    base_network                    1              1              1
-    build_bus_regions               1              1              1
-    build_hydro_profile             1              1              1
-    build_electricity_demand        1              1              1
-    build_powerplants               1              1              1
-    build_renewable_profiles        4              1              1
-    build_shapes                    1              1              1
-    build_ship_raster               1              1              1
-    cluster_network                 1              1              1
-    prepare_network                 1              1              1
-    retrieve_cost_data              1              1              1
-    retrieve_databundle             1              1              1
-    retrieve_natura_raster          1              1              1
-    simplify_network                1              1              1
-    solve_network                   1              1              1
-    total                          20              1              1
+    Job stats:
+    job                            count
+    ---------------------------  -------
+    add_electricity                    1
+    add_extra_components               1
+    base_network                       1
+    build_bus_regions                  1
+    build_electricity_demand           1
+    build_line_rating                  1
+    build_powerplants                  1
+    build_renewable_profiles           4
+    build_shapes                       1
+    build_ship_raster                  1
+    cluster_network                    1
+    copy_config                        1
+    prepare_network                    1
+    retrieve_cost_data                 1
+    retrieve_databundle                1
+    retrieve_electricity_demand        1
+    retrieve_natura_raster             1
+    simplify_network                   1
+    solve_network                      1
+    total                             22
 
 
 ``snakemake`` then runs these jobs in the correct order.
@@ -258,16 +268,16 @@ A job (here ``simplify_network``) will display its attributes and normally some
 
 .. code:: bash
 
-    [Mon Jan 1 00:00:00 2023]
+    [Mon Feb 19 17:06:17 2024]
     rule simplify_network:
-        input: networks/elec.nc, resources/costs.csv, resources/regions_onshore.geojson, resources/regions_offshore.geojson
-        output: networks/elec_s.nc, resources/regions_onshore_elec_s.geojson, resources/regions_offshore_elec_s.geojson, resources/busmap_elec_s.csv, resources/connection_costs_s.csv
-        log: logs/simplify_network/elec_s.log
+        input: resources/test/networks/elec.nc, data/costs_2030.csv, resources/test/regions_onshore.geojson, resources/test/regions_offshore.geojson
+        output: resources/test/networks/elec_s.nc, resources/test/regions_onshore_elec_s.geojson, resources/test/regions_offshore_elec_s.geojson, resources/test/busmap_elec_s.csv, resources/test/connection_costs_s.csv
+        log: logs/test-elec/simplify_network/elec_s.log
         jobid: 4
-        benchmark: benchmarks/simplify_network/elec_s
-        reason: Missing output files: resources/busmap_elec_s.csv, resources/regions_onshore_elec_s.geojson, networks/elec_s.nc, resources/regions_offshore_elec_s.geojson; Input files updated by another job: resources/regions_offshore.geojson, resources/regions_onshore.geojson, resources/costs.csv, networks/elec.nc
+        benchmark: benchmarks/test-elec/simplify_network/elec_s
+        reason: Missing output files: resources/test/regions_offshore_elec_s.geojson, resources/test/busmap_elec_s.csv, resources/test/regions_onshore_elec_s.geojson, resources/test/networks/elec_s.nc; Input files updated by another job: resources/test/regions_offshore.geojson, resources/test/networks/elec.nc, resources/test/regions_onshore.geojson, data/costs_2030.csv
         wildcards: simpl=
-        resources: tmpdir=/tmp, mem_mb=4000, mem_mib=3815
+        resources: tmpdir=/tmp, mem_mb=12000, mem_mib=11445
 
 Once the whole worktree is finished, it should state so in the terminal.
 

doc/validation.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 

doc/wildcards.rst

@@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors
 
   SPDX-License-Identifier: CC-BY-4.0
 

envs/environment.fixed.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 
@@ -12,94 +12,90 @@ dependencies:
 - _libgcc_mutex=0.1
 - _openmp_mutex=4.5
 - affine=2.4.0
-- alsa-lib=1.2.9
+- alsa-lib=1.2.10
 - ampl-mp=3.1.0
 - amply=0.1.6
-- anyio=3.7.1
 - appdirs=1.4.4
-- argon2-cffi=21.3.0
-- argon2-cffi-bindings=21.2.0
-- asttokens=2.2.1
-- async-lru=2.0.3
+- asttokens=2.4.1
 - atk-1.0=2.38.0
-- atlite=0.2.11
+- atlite=0.2.12
 - attr=2.5.1
-- attrs=23.1.0
-- aws-c-auth=0.7.0
-- aws-c-cal=0.6.0
-- aws-c-common=0.8.23
+- attrs=23.2.0
+- aws-c-auth=0.7.15
+- aws-c-cal=0.6.9
+- aws-c-common=0.9.12
 - aws-c-compression=0.2.17
-- aws-c-event-stream=0.3.1
-- aws-c-http=0.7.11
-- aws-c-io=0.13.28
-- aws-c-mqtt=0.8.14
-- aws-c-s3=0.3.13
-- aws-c-sdkutils=0.1.11
-- aws-checksums=0.1.16
-- aws-crt-cpp=0.20.3
-- aws-sdk-cpp=1.10.57
-- babel=2.12.1
-- backcall=0.2.0
-- backports=1.0
-- backports.functools_lru_cache=1.6.5
-- beautifulsoup4=4.12.2
-- bleach=6.0.0
-- blosc=1.21.4
-- bokeh=3.2.1
-- boost-cpp=1.78.0
+- aws-c-event-stream=0.4.1
+- aws-c-http=0.8.0
+- aws-c-io=0.14.3
+- aws-c-mqtt=0.10.1
+- aws-c-s3=0.5.0
+- aws-c-sdkutils=0.1.14
+- aws-checksums=0.1.17
+- aws-crt-cpp=0.26.1
+- aws-sdk-cpp=1.11.242
+- azure-core-cpp=1.10.3
+- azure-storage-blobs-cpp=12.10.0
+- azure-storage-common-cpp=12.5.0
+- beautifulsoup4=4.12.3
+- blosc=1.21.5
+- bokeh=3.3.4
 - bottleneck=1.3.7
-- branca=0.6.0
-- brotli=1.0.9
-- brotli-bin=1.0.9
-- brotli-python=1.0.9
+- branca=0.7.1
+- brotli=1.1.0
+- brotli-bin=1.1.0
+- brotli-python=1.1.0
 - bzip2=1.0.8
-- c-ares=1.19.1
-- c-blosc2=2.10.0
-- ca-certificates=2023.7.22
-- cairo=1.16.0
-- cartopy=0.21.1
+- c-ares=1.26.0
+- c-blosc2=2.13.2
+- ca-certificates=2024.2.2
+- cairo=1.18.0
+- cartopy=0.22.0
 - cdsapi=0.6.1
-- certifi=2023.7.22
-- cffi=1.15.1
-- cfitsio=4.2.0
-- cftime=1.6.2
-- charset-normalizer=3.2.0
-- click=8.1.6
+- certifi=2024.2.2
+- cffi=1.16.0
+- cfgv=3.3.1
+- cfitsio=4.3.1
+- cftime=1.6.3
+- charset-normalizer=3.3.2
+- click=8.1.7
 - click-plugins=1.1.1
 - cligj=0.7.2
-- cloudpickle=2.2.1
+- cloudpickle=3.0.0
+- coin-or-cbc=2.10.10
+- coin-or-cgl=0.60.7
+- coin-or-clp=1.17.8
+- coin-or-osi=0.108.8
+- coin-or-utils=2.11.9
+- coincbc=2.10.10
 - colorama=0.4.6
-- comm=0.1.3
 - configargparse=1.7
 - connection_pool=0.0.3
-- contourpy=1.1.0
-- country_converter=1.0.0
-- curl=8.2.0
-- cycler=0.11.0
-- cytoolz=0.12.2
-- dask=2023.7.1
-- dask-core=2023.7.1
+- contourpy=1.2.0
+- country_converter=1.2
+- cppad=20240000.2
+- cycler=0.12.1
+- cytoolz=0.12.3
+- dask=2024.2.0
+- dask-core=2024.2.0
 - datrie=0.8.2
 - dbus=1.13.6
-- debugpy=1.6.7
 - decorator=5.1.1
-- defusedxml=0.7.1
 - deprecation=2.1.0
 - descartes=1.1.0
-- distributed=2023.7.1
-- distro=1.8.0
+- distlib=0.3.8
+- distributed=2024.2.0
+- distro=1.9.0
 - docutils=0.20.1
 - dpath=2.1.6
-- entrypoints=0.4
-- entsoe-py=0.5.10
+- entsoe-py=0.6.6
 - et_xmlfile=1.1.0
-- exceptiongroup=1.1.2
-- executing=1.2.0
+- exceptiongroup=1.2.0
+- executing=2.0.1
 - expat=2.5.0
-- filelock=3.12.2
-- fiona=1.9.4
-- flit-core=3.9.0
-- folium=0.14.0
+- filelock=3.13.1
+- fiona=1.9.5
+- folium=0.15.1
 - font-ttf-dejavu-sans-mono=2.37
 - font-ttf-inconsolata=3.000
 - font-ttf-source-code-pro=2.038
@@ -107,366 +103,344 @@ dependencies:
 - fontconfig=2.14.2
 - fonts-conda-ecosystem=1
 - fonts-conda-forge=1
-- fonttools=4.41.1
+- fonttools=4.49.0
 - freetype=2.12.1
-- freexl=1.0.6
+- freexl=2.0.0
 - fribidi=1.0.10
-- fsspec=2023.6.0
-- gdal=3.7.0
+- fsspec=2024.2.0
+- gdal=3.8.4
 - gdk-pixbuf=2.42.10
 - geographiclib=1.52
-- geojson-rewind=1.0.2
-- geopandas=0.13.2
-- geopandas-base=0.13.2
-- geopy=2.3.0
-- geos=3.11.2
+- geojson-rewind=1.1.0
+- geopandas=0.14.3
+- geopandas-base=0.14.3
+- geopy=2.4.1
+- geos=3.12.1
 - geotiff=1.7.1
 - gettext=0.21.1
 - gflags=2.2.2
 - giflib=5.2.1
-- gitdb=4.0.10
-- gitpython=3.1.32
-- glib=2.76.4
-- glib-tools=2.76.4
+- gitdb=4.0.11
+- gitpython=3.1.42
+- glib=2.78.4
+- glib-tools=2.78.4
 - glog=0.6.0
-- gmp=6.2.1
+- glpk=5.0
+- gmp=6.3.0
 - graphite2=1.3.13
-- graphviz=8.1.0
-- gst-plugins-base=1.22.5
-- gstreamer=1.22.5
+- graphviz=9.0.0
+- gst-plugins-base=1.22.9
+- gstreamer=1.22.9
 - gtk2=2.24.33
 - gts=0.7.6
-- harfbuzz=7.3.0
+- harfbuzz=8.3.0
 - hdf4=4.2.15
-- hdf5=1.14.1
+- hdf5=1.14.3
 - humanfriendly=10.0
-- icu=72.1
-- idna=3.4
-- importlib-metadata=6.8.0
-- importlib_metadata=6.8.0
-- importlib_resources=6.0.0
+- icu=73.2
+- identify=2.5.35
+- idna=3.6
+- importlib-metadata=7.0.1
+- importlib_metadata=7.0.1
+- importlib_resources=6.1.1
 - iniconfig=2.0.0
-- ipopt=3.14.12
-- ipykernel=6.24.0
-- ipython=8.14.0
-- ipython_genutils=0.2.0
-- ipywidgets=8.0.7
-- jedi=0.18.2
-- jinja2=3.1.2
-- joblib=1.3.0
-- json-c=0.16
-- json5=0.9.14
-- jsonschema=4.18.4
-- jsonschema-specifications=2023.7.1
-- jupyter=1.0.0
-- jupyter-lsp=2.2.0
-- jupyter_client=8.3.0
-- jupyter_console=6.6.3
-- jupyter_core=5.3.1
-- jupyter_events=0.6.3
-- jupyter_server=2.7.0
-- jupyter_server_terminals=0.4.4
-- jupyterlab=4.0.3
-- jupyterlab_pygments=0.2.2
-- jupyterlab_server=2.24.0
-- jupyterlab_widgets=3.0.8
-- kealib=1.5.1
+- ipopt=3.14.14
+- ipython=8.21.0
+- jedi=0.19.1
+- jinja2=3.1.3
+- joblib=1.3.2
+- json-c=0.17
+- jsonschema=4.21.1
+- jsonschema-specifications=2023.12.1
+- jupyter_core=5.7.1
+- kealib=1.5.3
 - keyutils=1.6.1
-- kiwisolver=1.4.4
-- krb5=1.21.1
+- kiwisolver=1.4.5
+- krb5=1.21.2
 - lame=3.100
-- lcms2=2.15
+- lcms2=2.16
 - ld_impl_linux-64=2.40
 - lerc=4.0.0
-- libabseil=20230125.3
-- libaec=1.0.6
-- libarchive=3.6.2
-- libarrow=12.0.1
+- libabseil=20230802.1
+- libaec=1.1.2
+- libarchive=3.7.2
+- libarrow=15.0.0
+- libarrow-acero=15.0.0
+- libarrow-dataset=15.0.0
+- libarrow-flight=15.0.0
+- libarrow-flight-sql=15.0.0
+- libarrow-gandiva=15.0.0
+- libarrow-substrait=15.0.0
 - libblas=3.9.0
-- libbrotlicommon=1.0.9
-- libbrotlidec=1.0.9
-- libbrotlienc=1.0.9
-- libcap=2.67
+- libboost-headers=1.84.0
+- libbrotlicommon=1.1.0
+- libbrotlidec=1.1.0
+- libbrotlienc=1.1.0
+- libcap=2.69
 - libcblas=3.9.0
 - libclang=15.0.7
 - libclang13=15.0.7
 - libcrc32c=1.1.2
 - libcups=2.3.3
-- libcurl=8.2.0
-- libdeflate=1.18
+- libcurl=8.5.0
+- libdeflate=1.19
 - libedit=3.1.20191231
 - libev=4.33
 - libevent=2.1.12
 - libexpat=2.5.0
 - libffi=3.4.2
 - libflac=1.4.3
-- libgcc-ng=13.1.0
-- libgcrypt=1.10.1
+- libgcc-ng=13.2.0
+- libgcrypt=1.10.3
 - libgd=2.3.3
-- libgdal=3.7.0
-- libgfortran-ng=13.1.0
-- libgfortran5=13.1.0
-- libglib=2.76.4
-- libgomp=13.1.0
+- libgdal=3.8.4
+- libgfortran-ng=13.2.0
+- libgfortran5=13.2.0
+- libglib=2.78.4
+- libgomp=13.2.0
 - libgoogle-cloud=2.12.0
 - libgpg-error=1.47
-- libgrpc=1.56.2
+- libgrpc=1.60.1
+- libhwloc=2.9.3
 - libiconv=1.17
-- libjpeg-turbo=2.1.5.1
+- libjpeg-turbo=3.0.0
 - libkml=1.3.0
 - liblapack=3.9.0
 - liblapacke=3.9.0
 - libllvm15=15.0.7
 - libnetcdf=4.9.2
-- libnghttp2=1.52.0
-- libnsl=2.0.0
+- libnghttp2=1.58.0
+- libnl=3.9.0
+- libnsl=2.0.1
 - libnuma=2.0.16
 - libogg=1.3.4
-- libopenblas=0.3.23
+- libopenblas=0.3.26
 - libopus=1.3.1
-- libpng=1.6.39
-- libpq=15.3
-- libprotobuf=4.23.3
-- librsvg=2.56.1
+- libparquet=15.0.0
+- libpng=1.6.42
+- libpq=16.2
+- libprotobuf=4.25.1
+- libre2-11=2023.06.02
+- librsvg=2.56.3
 - librttopo=1.1.0
-- libsndfile=1.2.0
-- libsodium=1.0.18
+- libscotch=7.0.4
+- libsndfile=1.2.2
 - libspatialindex=1.9.3
-- libspatialite=5.0.1
-- libsqlite=3.42.0
+- libspatialite=5.1.0
+- libspral=2023.09.07
+- libsqlite=3.45.1
 - libssh2=1.11.0
-- libstdcxx-ng=13.1.0
-- libsystemd0=253
-- libthrift=0.18.1
-- libtiff=4.5.1
-- libtool=2.4.7
+- libstdcxx-ng=13.2.0
+- libsystemd0=255
+- libthrift=0.19.0
+- libtiff=4.6.0
 - libutf8proc=2.8.0
 - libuuid=2.38.1
 - libvorbis=1.3.7
-- libwebp=1.3.1
-- libwebp-base=1.3.1
+- libwebp=1.3.2
+- libwebp-base=1.3.2
 - libxcb=1.15
-- libxkbcommon=1.5.0
-- libxml2=2.11.4
-- libxslt=1.1.37
-- libzip=1.9.2
+- libxcrypt=4.4.36
+- libxkbcommon=1.6.0
+- libxml2=2.12.5
+- libxslt=1.1.39
+- libzip=1.10.1
 - libzlib=1.2.13
+- linopy=0.3.4
 - locket=1.0.0
-- lxml=4.9.3
-- lz4=4.3.2
+- lxml=5.1.0
+- lz4=4.3.3
 - lz4-c=1.9.4
 - lzo=2.10
-- mapclassify=2.5.0
-- markupsafe=2.1.3
-- matplotlib=3.5.3
-- matplotlib-base=3.5.3
+- mapclassify=2.6.1
+- markupsafe=2.1.5
+- matplotlib=3.8.3
+- matplotlib-base=3.8.3
 - matplotlib-inline=0.1.6
 - memory_profiler=0.61.0
-- metis=5.1.1
-- mistune=3.0.0
-- mpg123=1.31.3
-- msgpack-python=1.0.5
-- mumps-include=5.2.1
-- mumps-seq=5.2.1
-- munch=4.0.0
+- metis=5.1.0
+- minizip=4.0.4
+- mpg123=1.32.4
+- msgpack-python=1.0.7
+- mumps-include=5.6.2
+- mumps-seq=5.6.2
 - munkres=1.1.4
 - mysql-common=8.0.33
 - mysql-libs=8.0.33
-- nbclient=0.8.0
-- nbconvert=7.7.2
-- nbconvert-core=7.7.2
-- nbconvert-pandoc=7.7.2
-- nbformat=5.9.1
+- nbformat=5.9.2
 - ncurses=6.4
-- nest-asyncio=1.5.6
-- netcdf4=1.6.4
-- networkx=3.1
+- netcdf4=1.6.5
+- networkx=3.2.1
+- nodeenv=1.8.0
 - nomkl=1.0
-- notebook=7.0.0
-- notebook-shim=0.2.3
 - nspr=4.35
-- nss=3.89
-- numexpr=2.8.4
-- numpy=1.25.1
-- openjdk=17.0.3
+- nss=3.98
+- numexpr=2.9.0
+- numpy=1.26.4
+- openjdk=21.0.2
 - openjpeg=2.5.0
 - openpyxl=3.1.2
-- openssl=3.1.1
-- orc=1.9.0
-- overrides=7.3.1
-- packaging=23.1
-- pandas=2.0.3
-- pandoc=3.1.3
-- pandocfilters=1.5.0
+- openssl=3.2.1
+- orc=1.9.2
+- packaging=23.2
+- pandas=2.2.0
 - pango=1.50.14
 - parso=0.8.3
-- partd=1.4.0
-- patsy=0.5.3
-- pcre2=10.40
-- pexpect=4.8.0
+- partd=1.4.1
+- patsy=0.5.6
+- pcre2=10.42
+- pexpect=4.9.0
 - pickleshare=0.7.5
-- pillow=10.0.0
-- pip=23.2.1
-- pixman=0.40.0
+- pillow=10.2.0
+- pip=24.0
+- pixman=0.43.2
 - pkgutil-resolve-name=1.3.10
-- plac=1.3.5
-- platformdirs=3.9.1
-- pluggy=1.2.0
+- plac=1.4.2
+- platformdirs=4.2.0
+- pluggy=1.4.0
 - ply=3.11
-- pooch=1.7.0
-- poppler=23.05.0
+- poppler=24.02.0
 - poppler-data=0.4.12
-- postgresql=15.3
-- powerplantmatching=0.5.7
-- progressbar2=4.2.0
-- proj=9.2.1
-- prometheus_client=0.17.1
-- prompt-toolkit=3.0.39
-- prompt_toolkit=3.0.39
-- psutil=5.9.5
+- postgresql=16.2
+- powerplantmatching=0.5.11
+- pre-commit=3.6.2
+- progressbar2=4.3.2
+- proj=9.3.1
+- prompt-toolkit=3.0.42
+- psutil=5.9.8
 - pthread-stubs=0.4
 - ptyprocess=0.7.0
 - pulp=2.7.0
 - pulseaudio-client=16.1
 - pure_eval=0.2.2
 - py-cpuinfo=9.0.0
-- pyarrow=12.0.1
+- pyarrow=15.0.0
+- pyarrow-hotfix=0.6
 - pycountry=22.3.5
 - pycparser=2.21
-- pygments=2.15.1
+- pygments=2.17.2
 - pyomo=6.6.1
-- pyparsing=3.1.0
-- pyproj=3.6.0
-- pyqt=5.15.7
-- pyqt5-sip=12.11.0
+- pyparsing=3.1.1
+- pyproj=3.6.1
+- pypsa=0.27.0
+- pyqt=5.15.9
+- pyqt5-sip=12.12.2
+- pyscipopt=4.4.0
 - pyshp=2.3.1
 - pysocks=1.7.1
-- pytables=3.8.0
-- pytest=7.4.0
-- python=3.10.12
+- pytables=3.9.2
+- pytest=8.0.0
+- python=3.11.8
 - python-dateutil=2.8.2
-- python-fastjsonschema=2.18.0
-- python-json-logger=2.0.7
-- python-tzdata=2023.3
-- python-utils=3.7.0
-- python_abi=3.10
-- pytz=2023.3
+- python-fastjsonschema=2.19.1
+- python-tzdata=2024.1
+- python-utils=3.8.2
+- python_abi=3.11
+- pytz=2024.1
 - pyxlsb=1.0.10
-- pyyaml=6.0
-- pyzmq=25.1.0
+- pyyaml=6.0.1
 - qt-main=5.15.8
-- qtconsole=5.4.3
-- qtconsole-base=5.4.3
-- qtpy=2.3.1
-- rasterio=1.3.8
-- rdma-core=28.9
-- re2=2023.03.02
+- rasterio=1.3.9
+- rdma-core=50.0
+- re2=2023.06.02
 - readline=8.2
-- referencing=0.30.0
+- referencing=0.33.0
 - requests=2.31.0
 - reretry=0.11.8
-- rfc3339-validator=0.1.4
-- rfc3986-validator=0.1.1
-- rioxarray=0.14.1
-- rpds-py=0.9.2
-- rtree=1.0.1
-- s2n=1.3.46
-- scikit-learn=1.3.0
-- scipy=1.11.1
-- scotch=6.0.9
-- seaborn=0.12.2
-- seaborn-base=0.12.2
-- send2trash=1.8.2
-- setuptools=68.0.0
-- setuptools-scm=7.1.0
-- setuptools_scm=7.1.0
-- shapely=2.0.1
-- sip=6.7.10
+- rioxarray=0.15.1
+- rpds-py=0.18.0
+- rtree=1.2.0
+- s2n=1.4.3
+- scikit-learn=1.4.1.post1
+- scip=8.1.0
+- scipy=1.12.0
+- scotch=7.0.4
+- seaborn=0.13.2
+- seaborn-base=0.13.2
+- setuptools=69.1.0
+- setuptools-scm=8.0.4
+- setuptools_scm=8.0.4
+- shapely=2.0.2
+- sip=6.7.12
 - six=1.16.0
-- smart_open=6.3.0
-- smmap=3.0.5
-- snakemake-minimal=7.30.2
+- smart_open=6.4.0
+- smmap=5.0.0
+- snakemake-minimal=7.32.4
 - snappy=1.1.10
-- sniffio=1.3.0
 - snuggs=1.4.7
 - sortedcontainers=2.4.0
-- soupsieve=2.3.2.post1
-- sqlite=3.42.0
+- soupsieve=2.5
+- sqlite=3.45.1
 - stack_data=0.6.2
-- statsmodels=0.14.0
+- statsmodels=0.14.1
 - stopit=1.1.2
-- tabula-py=2.6.0
+- tabula-py=2.7.0
 - tabulate=0.9.0
-- tblib=1.7.0
-- terminado=0.17.1
-- threadpoolctl=3.2.0
-- throttler=1.2.1
-- tiledb=2.13.2
-- tinycss2=1.2.1
-- tk=8.6.12
+- tbb=2021.11.0
+- tblib=3.0.0
+- threadpoolctl=3.3.0
+- throttler=1.2.2
+- tiledb=2.20.0
+- tk=8.6.13
 - toml=0.10.2
 - tomli=2.0.1
-- toolz=0.12.0
+- toolz=0.12.1
 - toposort=1.10
-- tornado=6.3.2
-- tqdm=4.65.0
-- traitlets=5.9.0
-- typing-extensions=4.7.1
-- typing_extensions=4.7.1
-- typing_utils=0.1.0
-- tzcode=2023c
-- tzdata=2023c
-- ucx=1.14.1
-- unicodedata2=15.0.0
-- unidecode=1.3.6
-- unixodbc=2.3.10
-- urllib3=2.0.4
-- wcwidth=0.2.6
-- webencodings=0.5.1
-- websocket-client=1.6.1
-- wheel=0.41.0
-- widgetsnbextension=4.0.8
-- wrapt=1.15.0
-- xarray=2023.7.0
+- tornado=6.3.3
+- tqdm=4.66.2
+- traitlets=5.14.1
+- typing-extensions=4.9.0
+- typing_extensions=4.9.0
+- tzcode=2024a
+- tzdata=2024a
+- ucx=1.15.0
+- ukkonen=1.0.1
+- unidecode=1.3.8
+- unixodbc=2.3.12
+- uriparser=0.9.7
+- urllib3=2.2.1
+- validators=0.22.0
+- virtualenv=20.25.0
+- wcwidth=0.2.13
+- wheel=0.42.0
+- wrapt=1.16.0
+- xarray=2024.2.0
 - xcb-util=0.4.0
 - xcb-util-image=0.4.0
 - xcb-util-keysyms=0.4.0
 - xcb-util-renderutil=0.3.9
 - xcb-util-wm=0.4.1
-- xerces-c=3.2.4
-- xkeyboard-config=2.39
+- xerces-c=3.2.5
+- xkeyboard-config=2.41
 - xlrd=2.0.1
 - xorg-fixesproto=5.0
 - xorg-inputproto=2.3.2
 - xorg-kbproto=1.0.7
 - xorg-libice=1.1.1
 - xorg-libsm=1.2.4
-- xorg-libx11=1.8.6
+- xorg-libx11=1.8.7
 - xorg-libxau=1.0.11
 - xorg-libxdmcp=1.1.3
 - xorg-libxext=1.3.4
 - xorg-libxfixes=5.0.3
 - xorg-libxi=1.7.10
 - xorg-libxrender=0.9.11
+- xorg-libxt=1.3.0
 - xorg-libxtst=1.2.3
 - xorg-recordproto=1.14.2
 - xorg-renderproto=0.11.1
 - xorg-xextproto=7.3.0
 - xorg-xf86vidmodeproto=2.3.1
 - xorg-xproto=7.0.31
-- xyzservices=2023.7.0
+- xyzservices=2023.10.1
 - xz=5.2.6
 - yaml=0.2.5
-- yte=1.5.1
-- zeromq=4.3.4
+- yte=1.5.4
 - zict=3.0.0
-- zipp=3.16.2
+- zipp=3.17.0
 - zlib=1.2.13
 - zlib-ng=2.0.7
-- zstd=1.5.2
+- zstd=1.5.5
 - pip:
-  - gurobipy==10.0.2
-  - linopy==0.2.2
-  - pypsa==0.25.1
-  - tsam==2.3.0
-  - validators==0.20.0
+  - highspy==1.5.3
+  - tsam==2.3.1

envs/environment.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 
@@ -26,7 +26,7 @@ dependencies:
 - yaml
 - pytables
 - lxml
-- powerplantmatching>=0.5.5
+- powerplantmatching>=0.5.5,!=0.5.9
 - numpy
 - pandas>=2.1
 - geopandas>=0.11.0
@@ -35,8 +35,9 @@ dependencies:
 - netcdf4
 - networkx
 - scipy
+- glpk
 - shapely>=2.0
-- pyomo
+- pyscipopt
 - matplotlib
 - proj
 - fiona
@@ -47,7 +48,7 @@ dependencies:
 - tabula-py
 - pyxlsb
 - graphviz
-- ipopt
+- pre-commit
 
   # Keep in conda environment when calling ipython
 - ipython
@@ -60,3 +61,4 @@ dependencies:
 
 - pip:
   - tsam>=2.3.1
+  - highspy

envs/retrieve.yaml

@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+name: pypsa-eur-retrieve
+channels:
+- conda-forge
+- bioconda
+dependencies:
+- python>=3.8
+- snakemake-minimal>=7.7.0,<8.0.0
+- pandas>=2.1
+- tqdm

matplotlibrc

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 font.family: sans-serif

rules/build_electricity.smk

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2023-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 
@@ -8,7 +8,7 @@ if config["enable"].get("prepare_links_p_nom", False):
         output:
             "data/links_p_nom.csv",
         log:
-            LOGS + "prepare_links_p_nom.log",
+            logs("prepare_links_p_nom.log"),
         threads: 1
         resources:
             mem_mb=1500,
@@ -19,18 +19,17 @@ if config["enable"].get("prepare_links_p_nom", False):
 
 
 if config["enable"].get("retrieve_opsd_load_data", True):
-
     rule build_electricity_demand:
         params:
-            snapshots=config["snapshots"],
-            countries=config["countries"],
-            load=config["load"],
+            snapshots=config_provider("snapshots"),
+            countries=config_provider("countries"),
+            load=config_provider("load"),
         input:
-            ancient(RESOURCES + "/load_raw.csv"),
+            ancient("data/electricity_demand_raw.csv"),
         output:
-            RESOURCES + "load.csv",
+            resources("electricity_demand{weather_year}.csv"),
         log:
-            LOGS + "build_electricity_demand.log",
+            logs("build_electricity_demand{weather_year}.log"),
         resources:
             mem_mb=5000,
         conda:
@@ -38,16 +37,16 @@ if config["enable"].get("retrieve_opsd_load_data", True):
         script:
             "../scripts/build_electricity_demand.py"
 
-
+            
 if config["enable"].get("retrieve_artificial_load_data", False):
 
     rule build_artificial_load_data:
         input:
             ancient("data/load_artificial_raw.csv"),
         output:
-            RESOURCES + "load{weather_year}.csv",
+            resources("electricity_demand{weather_year}.csv"),
         log:
-            LOGS + "build_artificial_load_data{weather_year}.log",
+            logs("build_artificial_load_data{weather_year}.log"),
         resources:
             mem_mb=5000,
         conda:
@@ -56,18 +55,22 @@ if config["enable"].get("retrieve_artificial_load_data", False):
             "../scripts/build_artificial_load_data.py"
 
 
+ruleorder: build_artificial_load_data > build_electricity_demand
+
+
 rule build_powerplants:
     params:
-        powerplants_filter=config["electricity"]["powerplants_filter"],
-        custom_powerplants=config["electricity"]["custom_powerplants"],
-        countries=config["countries"],
+        powerplants_filter=config_provider("electricity", "powerplants_filter"),
+        custom_powerplants=config_provider("electricity", "custom_powerplants"),
+        everywhere_powerplants=config_provider("electricity", "everywhere_powerplants"),
+        countries=config_provider("countries"),
     input:
-        base_network=RESOURCES + "networks/base.nc",
+        base_network=resources("networks/base.nc"),
         custom_powerplants="data/custom_powerplants.csv",
     output:
-        RESOURCES + "powerplants.csv",
+        resources("powerplants.csv"),
     log:
-        LOGS + "build_powerplants.log",
+        logs("build_powerplants.log"),
     threads: 1
     resources:
         mem_mb=5000,
@@ -79,11 +82,11 @@ rule build_powerplants:
 
 rule base_network:
     params:
-        countries=config["countries"],
-        snapshots=config["snapshots"],
-        lines=config["lines"],
-        links=config["links"],
-        transformers=config["transformers"],
+        countries=config_provider("countries"),
+        snapshots=config_provider("snapshots"),
+        lines=config_provider("lines"),
+        links=config_provider("links"),
+        transformers=config_provider("transformers"),
     input:
         eg_buses="data/entsoegridkit/buses.csv",
         eg_lines="data/entsoegridkit/lines.csv",
@@ -93,15 +96,15 @@ rule base_network:
         parameter_corrections="data/parameter_corrections.yaml",
         links_p_nom="data/links_p_nom.csv",
         links_tyndp="data/links_tyndp.csv",
-        country_shapes=RESOURCES + "country_shapes.geojson",
-        offshore_shapes=RESOURCES + "offshore_shapes.geojson",
-        europe_shape=RESOURCES + "europe_shape.geojson",
+        country_shapes=resources("country_shapes.geojson"),
+        offshore_shapes=resources("offshore_shapes.geojson"),
+        europe_shape=resources("europe_shape.geojson"),
     output:
-        RESOURCES + "networks/base.nc",
+        resources("networks/base.nc"),
     log:
-        LOGS + "base_network.log",
+        logs("base_network.log"),
     benchmark:
-        BENCHMARKS + "base_network"
+        benchmarks("base_network")
     threads: 1
     resources:
         mem_mb=1500,
@@ -113,7 +116,7 @@ rule base_network:
 
 rule build_shapes:
     params:
-        countries=config["countries"],
+        countries=config_provider("countries"),
     input:
         naturalearth=ancient("data/bundle/naturalearth/ne_10m_admin_0_countries.shp"),
         eez=ancient("data/bundle/eez/World_EEZ_v8_2014.shp"),
@@ -123,12 +126,12 @@ rule build_shapes:
         ch_cantons=ancient("data/bundle/ch_cantons.csv"),
         ch_popgdp=ancient("data/bundle/je-e-21.03.02.xls"),
     output:
-        country_shapes=RESOURCES + "country_shapes.geojson",
-        offshore_shapes=RESOURCES + "offshore_shapes.geojson",
-        europe_shape=RESOURCES + "europe_shape.geojson",
-        nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
+        country_shapes=resources("country_shapes.geojson"),
+        offshore_shapes=resources("offshore_shapes.geojson"),
+        europe_shape=resources("europe_shape.geojson"),
+        nuts3_shapes=resources("nuts3_shapes.geojson"),
     log:
-        LOGS + "build_shapes.log",
+        logs("build_shapes.log"),
     threads: 1
     resources:
         mem_mb=1500,
@@ -140,16 +143,16 @@ rule build_shapes:
 
 rule build_bus_regions:
     params:
-        countries=config["countries"],
+        countries=config_provider("countries"),
     input:
-        country_shapes=RESOURCES + "country_shapes.geojson",
-        offshore_shapes=RESOURCES + "offshore_shapes.geojson",
-        base_network=RESOURCES + "networks/base.nc",
+        country_shapes=resources("country_shapes.geojson"),
+        offshore_shapes=resources("offshore_shapes.geojson"),
+        base_network=resources("networks/base.nc"),
     output:
-        regions_onshore=RESOURCES + "regions_onshore.geojson",
-        regions_offshore=RESOURCES + "regions_offshore.geojson",
+        regions_onshore=resources("regions_onshore.geojson"),
+        regions_offshore=resources("regions_offshore.geojson"),
     log:
-        LOGS + "build_bus_regions.log",
+        logs("build_bus_regions.log"),
     threads: 1
     resources:
         mem_mb=1000,
@@ -165,20 +168,20 @@ if config["enable"].get("build_cutout", False):
 
     rule build_cutout:
         params:
-            snapshots=config["snapshots"],
-            cutouts=config["atlite"]["cutouts"],
+            snapshots=config_provider("snapshots"),
+            cutouts=config_provider("atlite", "cutouts"),
         input:
-            regions_onshore=RESOURCES + "regions_onshore.geojson",
-            regions_offshore=RESOURCES + "regions_offshore.geojson",
+            regions_onshore=resources("regions_onshore.geojson"),
+            regions_offshore=resources("regions_offshore.geojson"),
         output:
             protected("cutouts/" + CDIR + "{cutout}.nc"),
         log:
-            "logs/" + CDIR + "build_cutout/{cutout}.log",
+            logs(CDIR + "build_cutout/{cutout}.log"),
         benchmark:
             "benchmarks/" + CDIR + "build_cutout_{cutout}"
-        threads: ATLITE_NPROCESSES
+        threads: config["atlite"].get("nprocesses", 4)
         resources:
-            mem_mb=ATLITE_NPROCESSES * 1000,
+            mem_mb=config["atlite"].get("nprocesses", 4) * 1000,
         conda:
             "../envs/environment.yaml"
         script:
@@ -190,9 +193,9 @@ if config["enable"].get("build_cutout", False):
         output:
             protected("cutouts/" + CDIR + "{cutout}-{weather_year}.nc"),
         log:
-            "logs/" + CDIR + "build_cutout/{cutout}-{weather_year}.log",
+            logs(CDIR + "build_cutout/{cutout}-{weather_year}.log"),
         benchmark:
-            "benchmarks/" + CDIR + "build_cutout_{cutout}-{weather_year}"
+            benchmarks(CDIR + "build_cutout_{cutout}-{weather_year}")
         threads: ATLITE_NPROCESSES
         resources:
             mem_mb=ATLITE_NPROCESSES * 1000,
@@ -207,10 +210,10 @@ if config["enable"].get("build_cutout", False):
 def cutouts_for_extent_input(wildcards):
     # We only need these cutouts in order to determine geographic
     # boundaries, so just pick the first weather year available
-    weather_year = config["scenario"]["weather_year"][0]
+    weather_year = config_provider("scenario", "weather_year")(wildcards)[0]
     cutouts = []
-    for k in config["electricity"]["renewable_carriers"]:
-        cutout = config["renewable"][k]["cutout"]
+    for k in config_provider("electricity", "renewable_carriers")(wildcards):
+        cutout = config_provider("renewable", k, "cutout")(wildcards)
         cutout = cutout.replace("{weather_year}", weather_year)
         cutouts.append(f"cutouts/{CDIR}{cutout}.nc")
     return cutouts
@@ -223,11 +226,11 @@ if config["enable"].get("build_natura_raster", False):
             natura=ancient("data/bundle/natura/Natura2000_end2015.shp"),
             cutouts=cutouts_for_extent_input,
         output:
-            RESOURCES + "natura.tiff",
+            resources("natura.tiff"),
         resources:
             mem_mb=5000,
         log:
-            LOGS + "build_natura_raster.log",
+            logs("build_natura_raster.log"),
         conda:
             "../envs/environment.yaml"
         script:
@@ -239,13 +242,13 @@ rule build_ship_raster:
         ship_density="data/shipdensity_global.zip",
         cutouts=cutouts_for_extent_input,
     output:
-        RESOURCES + "shipdensity_raster.tif",
+        resources("shipdensity_raster.tif"),
     log:
-        LOGS + "build_ship_raster.log",
+        logs("build_ship_raster.log"),
     resources:
         mem_mb=5000,
     benchmark:
-        BENCHMARKS + "build_ship_raster"
+        benchmarks("build_ship_raster")
     conda:
         "../envs/environment.yaml"
     script:
@@ -259,33 +262,33 @@ rule determine_availability_matrix_MD_UA:
         wdpa_marine="data/WDPA_WDOECM_marine.gpkg",
         gebco=lambda w: (
             "data/bundle/GEBCO_2014_2D.nc"
-            if "max_depth" in config["renewable"][w.technology].keys()
+            if config_provider("renewable", w.technology)(w).get("max_depth")
             else []
         ),
         ship_density=lambda w: (
-            RESOURCES + "shipdensity_raster.tif"
-            if "ship_threshold" in config["renewable"][w.technology].keys()
+            resources("shipdensity_raster.tif")
+            if "ship_threshold" in config_provider("renewable", w.technology)(w).keys()
             else []
         ),
-        country_shapes=RESOURCES + "country_shapes.geojson",
-        offshore_shapes=RESOURCES + "offshore_shapes.geojson",
+        country_shapes=resources("country_shapes.geojson"),
+        offshore_shapes=resources("offshore_shapes.geojson"),
         regions=lambda w: (
-            RESOURCES + "regions_onshore.geojson"
+            resources("regions_onshore.geojson")
             if w.technology in ("onwind", "solar")
-            else RESOURCES + "regions_offshore.geojson"
+            else resources("regions_offshore.geojson")
         ),
         cutout=lambda w: "cutouts/"
         + CDIR
-        + config["renewable"][w.technology]["cutout"]
+        + config_provider("renewable", w.technology, "cutout")(w)
         + ".nc",
     output:
-        availability_matrix=RESOURCES + "availability_matrix_MD-UA_{technology}.nc",
-        availability_map=RESOURCES + "availability_matrix_MD-UA_{technology}.png",
+        availability_matrix=resources("availability_matrix_MD-UA_{technology}.nc"),
+        availability_map=resources("availability_matrix_MD-UA_{technology}.png"),
     log:
-        LOGS + "determine_availability_matrix_MD_UA_{technology}.log",
-    threads: ATLITE_NPROCESSES
+        logs("determine_availability_matrix_MD_UA_{technology}.log"),
+    threads: config["atlite"].get("nprocesses", 4)
     resources:
-        mem_mb=ATLITE_NPROCESSES * 5000,
+        mem_mb=config["atlite"].get("nprocesses", 4) * 5000,
     conda:
         "../envs/environment.yaml"
     script:
@@ -293,59 +296,67 @@ rule determine_availability_matrix_MD_UA:
 
 
 # Optional input when having Ukraine (UA) or Moldova (MD) in the countries list
-if {"UA", "MD"}.intersection(set(config["countries"])):
-    opt = {
-        "availability_matrix_MD_UA": RESOURCES
-        + "availability_matrix_MD-UA_{technology}.nc"
-    }
-else:
-    opt = {}
+def input_ua_md_availability_matrix(w):
+    countries = set(config_provider("countries")(w))
+    if {"UA", "MD"}.intersection(countries):
+        return {
+            "availability_matrix_MD_UA": resources(
+                "availability_matrix_MD-UA_{technology}.nc"
+            )
+        }
+    return {}
 
 
 rule build_renewable_profiles:
     params:
-        renewable=config["renewable"],
+        snapshots=config_provider("snapshots"),
+        renewable=config_provider("renewable"),
     input:
-        **opt,
-        base_network=RESOURCES + "networks/base.nc",
+        unpack(input_ua_md_availability_matrix),
+        base_network=resources("networks/base.nc"),
         corine=ancient("data/bundle/corine/g250_clc06_V18_5.tif"),
         natura=lambda w: (
-            RESOURCES + "natura.tiff"
-            if config["renewable"][w.technology]["natura"]
+            resources("natura.tiff")
+            if config_provider("renewable", w.technology, "natura")(w)
+            else []
+        ),
+        luisa=lambda w: (
+            "data/LUISA_basemap_020321_50m.tif"
+            if config_provider("renewable", w.technology, "luisa")(w)
             else []
         ),
         gebco=ancient(
             lambda w: (
                 "data/bundle/GEBCO_2014_2D.nc"
-                if config["renewable"][w.technology].get("max_depth")
+                if config_provider("renewable", w.technology)(w).get("max_depth")
                 else []
             )
         ),
         ship_density=lambda w: (
-            RESOURCES + "shipdensity_raster.tif"
-            if config["renewable"][w.technology].get("ship_threshold", False)
+            resources("shipdensity_raster.tif")
+            if "ship_threshold" in config_provider("renewable", w.technology)(w).keys()
             else []
         ),
-        country_shapes=RESOURCES + "country_shapes.geojson",
-        offshore_shapes=RESOURCES + "offshore_shapes.geojson",
+        country_shapes=resources("country_shapes.geojson"),
+        offshore_shapes=resources("offshore_shapes.geojson"),
         regions=lambda w: (
-            RESOURCES + "regions_onshore.geojson"
+            resources("regions_onshore.geojson")
             if w.technology in ("onwind", "solar")
-            else RESOURCES + "regions_offshore.geojson"
+            else resources("regions_offshore.geojson")
         ),
         cutout=lambda w: "cutouts/"
         + CDIR
-        + config["renewable"][w.technology]["cutout"]
+        + config_provider("renewable", w.technology, "cutout")(w)
         + ".nc",
     output:
-        profile=RESOURCES + "profile{weather_year}_{technology}.nc",
+        profile=resources("profile{weather_year}_{technology}.nc"),
     log:
-        LOGS + "build_renewable_profiles{weather_year}_{technology}.log",
+        logs("build_renewable_profile{weather_year}_{technology}.log"),
     benchmark:
-        BENCHMARKS + "build_renewable_profiles{weather_year}_{technology}"
-    threads: ATLITE_NPROCESSES
+        benchmarks("build_renewable_profiles{weather_year}_{technology}")
+    threads: config["atlite"].get("nprocesses", 4)
     resources:
-        mem_mb=ATLITE_NPROCESSES * 5000,
+        mem_mb=config["atlite"].get("nprocesses", 4) * 5000,
     wildcard_constraints:
         technology="(?!hydro).*",  # Any technology other than hydro
     conda:
@@ -359,10 +370,10 @@ rule build_monthly_prices:
         co2_price_raw="data/validation/emission-spot-primary-market-auction-report-2019-data.xls",
         fuel_price_raw="data/validation/energy-price-trends-xlsx-5619002.xlsx",
     output:
-        co2_price=RESOURCES + "co2_price.csv",
-        fuel_price=RESOURCES + "monthly_fuel_price.csv",
+        co2_price=resources("co2_price.csv"),
+        fuel_price=resources("monthly_fuel_price.csv"),
     log:
-        LOGS + "build_monthly_prices.log",
+        logs("build_monthly_prices.log"),
     threads: 1
     resources:
         mem_mb=5000,
@@ -374,19 +385,22 @@ rule build_monthly_prices:
 
 rule build_hydro_profile:
     params:
-        hydro=config["renewable"]["hydro"],
-        countries=config["countries"],
+        hydro=config_provider("renewable", "hydro"),
+        countries=config_provider("countries"),
     input:
-        country_shapes=RESOURCES + "country_shapes.geojson",
+        country_shapes=resources("country_shapes.geojson"),
         eia_hydro_generation="data/eia_hydro_annual_generation.csv",
         eia_hydro_capacity="data/eia_hydro_annual_capacity.csv",
         era5_runoff="data/era5-annual-runoff-per-country.csv",
-        cutout=f"cutouts/" + CDIR + config["renewable"]["hydro"]["cutout"] + ".nc",
+        cutout=lambda w: f"cutouts/"
+        + CDIR
+        + config_provider("renewable", "hydro", "cutout")(w)
+        + ".nc",
     output:
-        profile=RESOURCES + "profile{weather_year}_hydro.nc",
-        eia_hydro=RESOURCES + "eia_hydro_annual_generation{weather_year}.csv",
+        profile=resources("profile{weather_year}_hydro.nc"),
+        eia_hydro=resources("eia_hydro_stats{weather_year}.csv")
     log:
-        LOGS + "build_hydro_profile{weather_year}.log",
+        logs("build_hydro_profile{weather_year}.log"),
     resources:
         mem_mb=5000,
     conda:
@@ -395,73 +409,88 @@ rule build_hydro_profile:
         "../scripts/build_hydro_profile.py"
 
 
-if config["lines"]["dynamic_line_rating"]["activate"]:
+rule build_line_rating:
+    params:
+        snapshots=config_provider("snapshots"),
+    input:
+        base_network=resources("networks/base.nc"),
+        cutout=lambda w: "cutouts/"
+        + CDIR
+        + config_provider("lines", "dynamic_line_rating", "cutout")(w)
+        + ".nc",
+    output:
+        output=resources("networks/line_rating.nc"),
+    log:
+        logs("build_line_rating.log"),
+    benchmark:
+        benchmarks("build_line_rating")
+    threads: config["atlite"].get("nprocesses", 4)
+    resources:
+        mem_mb=config["atlite"].get("nprocesses", 4) * 1000,
+    conda:
+        "../envs/environment.yaml"
+    script:
+        "../scripts/build_line_rating.py"
 
-    rule build_line_rating:
-        input:
-            base_network=RESOURCES + "networks/base.nc",
-            cutout="cutouts/"
-            + CDIR
-            + config["lines"]["dynamic_line_rating"]["cutout"]
-            + ".nc",
-        output:
-            output=RESOURCES + "networks/line_rating.nc",
-        log:
-            LOGS + "build_line_rating.log",
-        benchmark:
-            BENCHMARKS + "build_line_rating"
-        threads: ATLITE_NPROCESSES
-        resources:
-            mem_mb=ATLITE_NPROCESSES * 1000,
-        conda:
-            "../envs/environment.yaml"
-        script:
-            "../scripts/build_line_rating.py"
+
+def input_profile_tech(w):
+    return {
+        f"profile_{tech}": resources(f"profile{{weather_year}}_{tech}.nc")
+        for tech in config_provider("electricity", "renewable_carriers")(w)
+    }
+
+
+def input_conventional(w):
+    return {
+        f"conventional_{carrier}_{attr}": fn
+        for carrier, d in config_provider("conventional", default={None: {}})(w).items()
+        if carrier in config_provider("electricity", "conventional_carriers")(w)
+        for attr, fn in d.items()
+        if str(fn).startswith("data/")
+    }
 
 
 rule add_electricity:
     params:
-        length_factor=config["lines"]["length_factor"],
-        scaling_factor=config["load"]["scaling_factor"],
-        countries=config["countries"],
-        renewable=config["renewable"],
-        electricity=config["electricity"],
-        conventional=config["conventional"],
-        costs=config["costs"],
+        length_factor=config_provider("lines", "length_factor"),
+        scaling_factor=config_provider("load", "scaling_factor"),
+        countries=config_provider("countries"),
+        renewable=config_provider("renewable"),
+        electricity=config_provider("electricity"),
+        conventional=config_provider("conventional"),
+        costs=config_provider("costs"),
+        drop_leap_day=config_provider("enable", "drop_leap_day"),
     input:
-        **{
-            f"profile_{tech}": RESOURCES + "profile{weather_year}" + f"_{tech}.nc"
-            for tech in config["electricity"]["renewable_carriers"]
-        },
-        **{
-            f"conventional_{carrier}_{attr}": fn
-            for carrier, d in config.get("conventional", {None: {}}).items()
-            if carrier in config["electricity"]["conventional_carriers"]
-            for attr, fn in d.items()
-            if str(fn).startswith("data/")
-        },
-        base_network=RESOURCES + "networks/base.nc",
-        line_rating=RESOURCES + "networks/line_rating.nc"
-        if config["lines"]["dynamic_line_rating"]["activate"]
-        else RESOURCES + "networks/base.nc",
-        tech_costs=COSTS,
-        regions=RESOURCES + "regions_onshore.geojson",
-        powerplants=RESOURCES + "powerplants.csv",
+        unpack(input_profile_tech),
+        unpack(input_conventional),
+        base_network=resources("networks/base.nc"),
+        line_rating=lambda w: (
+            resources("networks/line_rating.nc")
+            if config_provider("lines", "dynamic_line_rating", "activate")(w)
+            else resources("networks/base.nc")
+        ),
+        tech_costs=lambda w: resources(
+            f"costs_{config_provider('costs', 'year')(w)}.csv"
+        ),
+        regions=resources("regions_onshore.geojson"),
+        powerplants=resources("powerplants.csv"),
         hydro_capacities=ancient("data/bundle/hydro_capacities.csv"),
         geth_hydro_capacities="data/geth2015_hydro_capacities.csv",
         unit_commitment="data/unit_commitment.csv",
-        fuel_price=RESOURCES + "monthly_fuel_price.csv"
-        if config["conventional"]["dynamic_fuel_price"]
-        else [],
-        load=RESOURCES + "load{weather_year}.csv",
-        nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
+        fuel_price=lambda w: (
+            resources("monthly_fuel_price.csv")
+            if config_provider("conventional", "dynamic_fuel_price")(w)
+            else []
+        ),
+        load=resources("electricity_demand{weather_year}.csv"),
+        nuts3_shapes=resources("nuts3_shapes.geojson"),
         ua_md_gdp="data/GDP_PPP_30arcsec_v3_mapped_default.csv",
     output:
-        RESOURCES + "networks/elec{weather_year}.nc",
+        resources("networks/elec{weather_year}.nc"),
     log:
-        LOGS + "add_electricity{weather_year}.log",
+        logs("add_electricity{weather_year}.log"),
     benchmark:
-        BENCHMARKS + "add_electricity{weather_year}"
+        benchmarks("add_electricity{weather_year}")
     threads: 1
     resources:
         mem_mb=10000,
@@ -473,33 +502,33 @@ rule add_electricity:
 
 rule simplify_network:
     params:
-        simplify_network=config["clustering"]["simplify_network"],
-        aggregation_strategies=config["clustering"].get("aggregation_strategies", {}),
-        focus_weights=config["clustering"].get(
-            "focus_weights", config.get("focus_weights")
+        simplify_network=config_provider("clustering", "simplify_network"),
+        aggregation_strategies=config_provider(
+            "clustering", "aggregation_strategies", default={}
         ),
-        renewable_carriers=config["electricity"]["renewable_carriers"],
-        max_hours=config["electricity"]["max_hours"],
-        length_factor=config["lines"]["length_factor"],
-        p_max_pu=config["links"].get("p_max_pu", 1.0),
-        costs=config["costs"],
+        focus_weights=config_provider("clustering", "focus_weights", default=None),
+        renewable_carriers=config_provider("electricity", "renewable_carriers"),
+        max_hours=config_provider("electricity", "max_hours"),
+        length_factor=config_provider("lines", "length_factor"),
+        p_max_pu=config_provider("links", "p_max_pu", default=1.0),
+        costs=config_provider("costs"),
     input:
-        network=RESOURCES + "networks/elec{weather_year}.nc",
-        tech_costs=COSTS,
-        regions_onshore=RESOURCES + "regions_onshore.geojson",
-        regions_offshore=RESOURCES + "regions_offshore.geojson",
+        network=resources("networks/elec{weather_year}.nc"),
+        tech_costs=lambda w: resources(
+            f"costs_{config_provider('costs', 'year')(w)}.csv"
+        ),
+        regions_onshore=resources("regions_onshore.geojson"),
+        regions_offshore=resources("regions_offshore.geojson"),
     output:
-        network=RESOURCES + "networks/elec{weather_year}_s{simpl}.nc",
-        regions_onshore=RESOURCES
-        + "regions_onshore_elec{weather_year}_s{simpl}.geojson",
-        regions_offshore=RESOURCES
-        + "regions_offshore_elec{weather_year}_s{simpl}.geojson",
-        busmap=RESOURCES + "busmap_elec{weather_year}_s{simpl}.csv",
-        connection_costs=RESOURCES + "connection_costs{weather_year}_s{simpl}.csv",
+        network=resources("networks/elec{weather_year}_s{simpl}.nc"),
+        regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}.geojson"),
+        regions_offshore=resources("regions_offshore_elec{weather_year}_s{simpl}.geojson"),
+        busmap=resources("busmap_elec{weather_year}_s{simpl}.csv"),
+        connection_costs=resources("connection_costs{weather_year}_s{simpl}.csv"),
     log:
-        LOGS + "simplify_network/elec{weather_year}_s{simpl}.log",
+        logs("simplify_network/elec{weather_year}_s{simpl}.log"),
     benchmark:
-        BENCHMARKS + "simplify_network/elec{weather_year}_s{simpl}"
+        benchmarks("simplify_network/elec{weather_year}_s{simpl}")
     threads: 1
     resources:
         mem_mb=12000,
@@ -511,42 +540,42 @@ rule simplify_network:
 
 rule cluster_network:
     params:
-        cluster_network=config["clustering"]["cluster_network"],
-        aggregation_strategies=config["clustering"].get("aggregation_strategies", {}),
-        custom_busmap=config["enable"].get("custom_busmap", False),
-        focus_weights=config["clustering"].get(
-            "focus_weights", config.get("focus_weights")
+        cluster_network=config_provider("clustering", "cluster_network"),
+        aggregation_strategies=config_provider(
+            "clustering", "aggregation_strategies", default={}
         ),
-        renewable_carriers=config["electricity"]["renewable_carriers"],
-        conventional_carriers=config["electricity"].get("conventional_carriers", []),
-        max_hours=config["electricity"]["max_hours"],
-        length_factor=config["lines"]["length_factor"],
-        costs=config["costs"],
+        custom_busmap=config_provider("enable", "custom_busmap", default=False),
+        focus_weights=config_provider("clustering", "focus_weights", default=None),
+        renewable_carriers=config_provider("electricity", "renewable_carriers"),
+        conventional_carriers=config_provider(
+            "electricity", "conventional_carriers", default=[]
+        ),
+        max_hours=config_provider("electricity", "max_hours"),
+        length_factor=config_provider("lines", "length_factor"),
+        costs=config_provider("costs"),
     input:
-        network=RESOURCES + "networks/elec{weather_year}_s{simpl}.nc",
-        regions_onshore=RESOURCES
-        + "regions_onshore_elec{weather_year}_s{simpl}.geojson",
-        regions_offshore=RESOURCES
-        + "regions_offshore_elec{weather_year}_s{simpl}.geojson",
-        busmap=ancient(RESOURCES + "busmap_elec{weather_year}_s{simpl}.csv"),
-        custom_busmap=(
+        network=resources("networks/elec{weather_year}_s{simpl}.nc"),
+        regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}.geojson"),
+        regions_offshore=resources("regions_offshore_elec{weather_year}_s{simpl}.geojson"),
+        busmap=ancient(resources("busmap_elec{weather_year}_s{simpl}.csv")),
+        custom_busmap=lambda w: (
             "data/custom_busmap_elec_s{simpl}_{clusters}.csv"
-            if config["enable"].get("custom_busmap", False)
+            if config_provider("enable", "custom_busmap", default=False)(w)
             else []
         ),
-        tech_costs=COSTS,
+        tech_costs=lambda w: resources(
+            f"costs_{config_provider('costs', 'year')(w)}.csv"
+        ),
     output:
-        network=RESOURCES + "networks/elec{weather_year}_s{simpl}_{clusters}.nc",
-        regions_onshore=RESOURCES
-        + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson",
-        regions_offshore=RESOURCES
-        + "regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson",
-        busmap=RESOURCES + "busmap_elec{weather_year}_s{simpl}_{clusters}.csv",
-        linemap=RESOURCES + "linemap_elec{weather_year}_s{simpl}_{clusters}.csv",
+        network=resources("networks/elec{weather_year}_s{simpl}_{clusters}.nc"),
+        regions_onshore=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"),
+        regions_offshore=resources("regions_offshore_elec{weather_year}_s{simpl}_{clusters}.geojson"),
+        busmap=resources("busmap_elec{weather_year}_s{simpl}_{clusters}.csv"),
+        linemap=resources("linemap_elec{weather_year}_s{simpl}_{clusters}.csv"),
     log:
-        LOGS + "cluster_network/elec{weather_year}_s{simpl}_{clusters}.log",
+        logs("cluster_network/elec{weather_year}_s{simpl}_{clusters}.log"),
     benchmark:
-        BENCHMARKS + "cluster_network/elec{weather_year}_s{simpl}_{clusters}"
+        benchmarks("cluster_network/elec{weather_year}_s{simpl}_{clusters}")
     threads: 1
     resources:
         mem_mb=10000,
@@ -558,18 +587,20 @@ rule cluster_network:
 
 rule add_extra_components:
     params:
-        extendable_carriers=config["electricity"]["extendable_carriers"],
-        max_hours=config["electricity"]["max_hours"],
-        costs=config["costs"],
+        extendable_carriers=config_provider("electricity", "extendable_carriers"),
+        max_hours=config_provider("electricity", "max_hours"),
+        costs=config_provider("costs"),
     input:
-        network=RESOURCES + "networks/elec{weather_year}_s{simpl}_{clusters}.nc",
-        tech_costs=COSTS,
+        network=resources("networks/elec{weather_year}_s{simpl}_{clusters}.nc"),
+        tech_costs=lambda w: resources(
+            f"costs_{config_provider('costs', 'year')(w)}.csv"
+        ),
     output:
-        RESOURCES + "networks/elec{weather_year}_s{simpl}_{clusters}_ec.nc",
+        resources("networks/elec{weather_year}_s{simpl}_{clusters}_ec.nc"),
     log:
-        LOGS + "add_extra_components/elec{weather_year}_s{simpl}_{clusters}.log",
+        logs("add_extra_components/elec{weather_year}_s{simpl}_{clusters}.log"),
     benchmark:
-        BENCHMARKS + "add_extra_components/elec{weather_year}_s{simpl}_{clusters}_ec"
+        benchmarks("add_extra_components/elec{weather_year}_s{simpl}_{clusters}_ec")
     threads: 1
     resources:
         mem_mb=4000,
@@ -581,27 +612,30 @@ rule add_extra_components:
 
 rule prepare_network:
     params:
-        links=config["links"],
-        lines=config["lines"],
-        co2base=config["electricity"]["co2base"],
-        co2limit=config["electricity"]["co2limit"],
-        gaslimit=config["electricity"].get("gaslimit"),
-        max_hours=config["electricity"]["max_hours"],
-        costs=config["costs"],
+        time_resolution=config_provider("clustering", "temporal", "resolution_elec"),
+        links=config_provider("links"),
+        lines=config_provider("lines"),
+        co2base=config_provider("electricity", "co2base"),
+        co2limit_enable=config_provider("electricity", "co2limit_enable", default=False),
+        co2limit=config_provider("electricity", "co2limit"),
+        gaslimit_enable=config_provider("electricity", "gaslimit_enable", default=False),
+        gaslimit=config_provider("electricity", "gaslimit"),
+        max_hours=config_provider("electricity", "max_hours"),
+        costs=config_provider("costs"),
+        adjustments=config_provider("adjustments", "electricity"),
+        autarky=config_provider("electricity", "autarky", default={}),
     input:
-        RESOURCES + "networks/elec{weather_year}_s{simpl}_{clusters}_ec.nc",
-        tech_costs=COSTS,
-        co2_price=lambda w: RESOURCES + "co2_price.csv" if "Ept" in w.opts else [],
+        resources("networks/elec{weather_year}_s{simpl}_{clusters}_ec.nc"),
+        tech_costs=lambda w: resources(
+            f"costs_{config_provider('costs', 'year')(w)}.csv"
+        ),
+        co2_price=lambda w: resources("co2_price.csv") if "Ept" in w.opts else [],
     output:
-        RESOURCES + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
+        resources("networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"),
     log:
-        LOGS
-        + "prepare_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.log",
+        logs("prepare_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.log"),
     benchmark:
-        (
-            BENCHMARKS
-            + "prepare_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}"
-        )
+        (benchmarks("prepare_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}"))
     threads: 1
     resources:
         mem_mb=4000,

rules/collect.smk

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2023-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 
@@ -11,31 +11,32 @@ localrules:
     prepare_sector_networks,
     solve_elec_networks,
     solve_sector_networks,
-    plot_networks,
 
 
 rule cluster_networks:
     input:
         expand(
-            RESOURCES + "networks/elec{weather_year}_s{simpl}_{clusters}.nc",
-            **config["scenario"]
+            resources("networks/elec{weather_year}_s{simpl}_{clusters}.nc"),
+            **config["scenario"],
+            run=config["run"]["name"],
         ),
 
 
 rule extra_components_networks:
     input:
         expand(
-            RESOURCES + "networks/elec{weather_year}_s{simpl}_{clusters}_ec.nc",
-            **config["scenario"]
+            resources("networks/elec{weather_year}_s{simpl}_{clusters}_ec.nc"),
+            **config["scenario"],
+            run=config["run"]["name"],
         ),
 
 
 rule prepare_elec_networks:
     input:
         expand(
-            RESOURCES
-            + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
-            **config["scenario"]
+            resources("networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"),
+            **config["scenario"],
+            run=config["run"]["name"],
         ),
 
 
@@ -44,16 +45,17 @@ rule prepare_sector_networks:
         expand(
             RESULTS
             + "prenetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-            **config["scenario"]
+            **config["scenario"],
+            run=config["run"]["name"],
         ),
 
 
 rule solve_elec_networks:
     input:
         expand(
-            RESULTS
-            + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
-            **config["scenario"]
+            RESULTS + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
+            **config["scenario"],
+            run=config["run"]["name"],
         ),
 
 
@@ -62,25 +64,18 @@ rule solve_sector_networks:
         expand(
             RESULTS
             + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-            **config["scenario"]
+            **config["scenario"],
+            run=config["run"]["name"],
         ),
 
 
 rule solve_sector_networks_perfect:
-    input:
-        expand(
-            RESULTS
-            + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
-            **config["scenario"]
-        ),
-
-
-rule plot_networks:
     input:
         expand(
             RESULTS
             + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf",
-            **config["scenario"]
+            **config["scenario"],
+            run=config["run"]["name"],
         ),
 
 
@@ -88,12 +83,14 @@ rule validate_elec_networks:
     input:
         expand(
             RESULTS
-            + "figures/.statistics_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}",
-            **config["scenario"]
+            + "figures/.statistics_plots_elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}",
+            **config["scenario"],
+            run=config["run"]["name"],
         ),
         expand(
             RESULTS
-            + "figures/.validation_{kind}_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}",
+            + "figures/.validation_{kind}_plots_elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}",
             **config["scenario"],
-            kind=["production", "prices", "cross_border"]
+            run=config["run"]["name"],
+            kind=["production", "prices", "cross_border"],
         ),

rules/common.smk

@@ -1,11 +1,88 @@
-# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2023-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 
-import os, sys
+import copy
+from functools import partial, lru_cache
 
-sys.path.insert(0, os.path.abspath("scripts"))
-from _helpers import validate_checksum
+import os, sys, glob
+
+path = workflow.source_path("../scripts/_helpers.py")
+sys.path.insert(0, os.path.dirname(path))
+
+from _helpers import validate_checksum, update_config_from_wildcards
+from snakemake.utils import update_config
+
+
+def get_config(config, keys, default=None):
+    """Retrieve a nested value from a dictionary using a tuple of keys."""
+    value = config
+    for key in keys:
+        if isinstance(value, list):
+            value = value[key]
+        else:
+            value = value.get(key, default)
+        if value == default:
+            return default
+    return value
+
+
+def merge_configs(base_config, scenario_config):
+    """Merge base config with a specific scenario without modifying the original."""
+    merged = copy.deepcopy(base_config)
+    update_config(merged, scenario_config)
+    return merged
+
+
+@lru_cache
+def scenario_config(scenario_name):
+    """Retrieve a scenario config based on the overrides from the scenario file."""
+    return merge_configs(config, scenarios[scenario_name])
+
+
+def static_getter(wildcards, keys, default):
+    """Getter function for static config values."""
+    config_with_wildcards = update_config_from_wildcards(
+        config, wildcards, inplace=False
+    )
+    return get_config(config_with_wildcards, keys, default)
+
+
+def dynamic_getter(wildcards, keys, default):
+    """Getter function for dynamic config values based on scenario."""
+    if "run" not in wildcards.keys():
+        return get_config(config, keys, default)
+    scenario_name = wildcards.run
+    if scenario_name not in scenarios:
+        raise ValueError(
+            f"Scenario {scenario_name} not found in file {config['run']['scenario']['file']}."
+        )
+    config_with_scenario = scenario_config(scenario_name)
+    config_with_wildcards = update_config_from_wildcards(
+        config_with_scenario, wildcards, inplace=False
+    )
+    return get_config(config_with_wildcards, keys, default)
+
+
+def config_provider(*keys, default=None):
+    """Dynamically provide config values based on 'run' -> 'name'.
+
+    Usage in Snakemake rules would look something like:
+    params:
+        my_param=config_provider("key1", "key2", default="some_default_value")
+    """
+    # Using functools.partial to freeze certain arguments in our getter functions.
+    if config["run"].get("scenarios", {}).get("enable", False):
+        return partial(dynamic_getter, keys=keys, default=default)
+    else:
+        return partial(static_getter, keys=keys, default=default)
+
+
+def solver_threads(w):
+    solver_options = config_provider("solving", "solver_options")(w)
+    option_set = config_provider("solving", "solver", "options")(w)
+    threads = solver_options[option_set].get("threads", 4)
+    return threads
 
 
 def memory(w):
@@ -28,6 +105,15 @@ def memory(w):
         return int(factor * (10000 + 195 * int(w.clusters)))
 
 
+def input_custom_extra_functionality(w):
+    path = config_provider(
+        "solving", "options", "custom_extra_functionality", default=False
+    )(w)
+    if path:
+        return os.path.join(os.path.dirname(workflow.snakefile), path)
+    return []
+
+
 # Check if the workflow has access to the internet by trying to access the HEAD of specified url
 def has_internet_access(url="www.zenodo.org") -> bool:
     import http.client as http_client
@@ -46,14 +132,15 @@ def has_internet_access(url="www.zenodo.org") -> bool:
 
 def input_eurostat(w):
     # 2016 includes BA, 2017 does not
-    report_year = config["energy"]["eurostat_report_year"]
+    report_year = config_provider("energy", "eurostat_report_year")(w)
     return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition"
 
 
-def solved_previous_horizon(wildcards):
-    planning_horizons = config["scenario"]["planning_horizons"]
-    i = planning_horizons.index(int(wildcards.planning_horizons))
+def solved_previous_horizon(w):
+    planning_horizons = config_provider("scenario", "planning_horizons")(w)
+    i = planning_horizons.index(int(w.planning_horizons))
     planning_horizon_p = str(planning_horizons[i - 1])
+
     return (
         RESULTS
         + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_"

rules/postprocess.smk

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2023-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 
@@ -9,60 +9,132 @@ localrules:
 
 if config["foresight"] != "perfect":
 
-    rule plot_network:
+    rule plot_power_network_clustered:
         params:
-            foresight=config["foresight"],
-            plotting=config["plotting"],
+            plotting=config_provider("plotting"),
+        input:
+            network=resources("networks/elec{weather_year}_s{simpl}_{clusters}.nc"),
+            regions_onshore=resources(
+                "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"
+            ),
+        output:
+            map=resources("maps/power-network{weather_year}-s{simpl}-{clusters}.pdf"),
+        threads: 1
+        resources:
+            mem_mb=4000,
+        benchmark:
+            benchmarks("plot_power_network_clustered/elec{weather_year}_s{simpl}_{clusters}")
+        conda:
+            "../envs/environment.yaml"
+        script:
+            "../scripts/plot_power_network_clustered.py"
+
+    rule plot_power_network:
+        params:
+            plotting=config_provider("plotting"),
         input:
             network=RESULTS
             + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-            regions=RESOURCES + "regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson",
+            regions=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"),
         output:
             map=RESULTS
             + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf",
-            today=RESULTS
-            + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}-today.pdf",
         threads: 2
         resources:
             mem_mb=10000,
+        log:
+            RESULTS
+            + "logs/plot_power_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
         benchmark:
             (
-                BENCHMARKS
-                + "plot_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
+                RESULTS
+                + "benchmarksplot_power_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
             )
         conda:
             "../envs/environment.yaml"
         script:
-            "../scripts/plot_network.py"
+            "../scripts/plot_power_network.py"
+
+    rule plot_hydrogen_network:
+        params:
+            plotting=config_provider("plotting"),
+            foresight=config_provider("foresight"),
+        input:
+            network=RESULTS
+            + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
+            regions=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"),
+        output:
+            map=RESULTS
+            + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf",
+        threads: 2
+        resources:
+            mem_mb=10000,
+        log:
+            RESULTS
+            + "logs/plot_hydrogen_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
+        benchmark:
+            (
+                RESULTS
+                + "benchmarks/plot_hydrogen_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
+            )
+        conda:
+            "../envs/environment.yaml"
+        script:
+            "../scripts/plot_hydrogen_network.py"
+
+    rule plot_gas_network:
+        params:
+            plotting=config_provider("plotting"),
+        input:
+            network=RESULTS
+            + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
+            regions=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"),
+        output:
+            map=RESULTS
+            + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-ch4_network_{planning_horizons}.pdf",
+        threads: 2
+        resources:
+            mem_mb=10000,
+        log:
+            RESULTS
+            + "logs/plot_gas_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
+        benchmark:
+            (
+                RESULTS
+                + "benchmarks/plot_gas_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
+            )
+        conda:
+            "../envs/environment.yaml"
+        script:
+            "../scripts/plot_gas_network.py"
 
 
 if config["foresight"] == "perfect":
 
-    rule plot_network:
+    def output_map_year(w):
+        return {
+            f"map_{year}": RESULTS
+            + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_"
+            + f"{year}.pdf"
+            for year in config_provider("scenario", "planning_horizons")(w)
+        }
+
+    rule plot_power_network_perfect:
         params:
-            foresight=config["foresight"],
-            plotting=config["plotting"],
+            plotting=config_provider("plotting"),
         input:
             network=RESULTS
-            + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
-            regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
+            + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
+            regions=resources("regions_onshore_elec{weather_year}_s{simpl}_{clusters}.geojson"),
         output:
-            **{
-                f"map_{year}": RESULTS
-                + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_"
-                + f"{year}.pdf"
-                for year in config["scenario"]["planning_horizons"]
-            },
+            unpack(output_map_year),
         threads: 2
         resources:
             mem_mb=10000,
-        benchmark:
-            BENCHMARKS
-            +"postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_benchmark"
         conda:
             "../envs/environment.yaml"
         script:
-            "../scripts/plot_network.py"
+            "../scripts/plot_power_network_perfect.py"
 
 
 rule copy_config:
@@ -73,8 +145,6 @@ rule copy_config:
     threads: 1
     resources:
         mem_mb=1000,
-    benchmark:
-        BENCHMARKS + "copy_config"
     conda:
         "../envs/environment.yaml"
     script:
@@ -83,24 +153,57 @@ rule copy_config:
 
 rule make_summary:
     params:
-        foresight=config["foresight"],
-        costs=config["costs"],
-        snapshots=config["snapshots"],
-        scenario=config["scenario"],
+        foresight=config_provider("foresight"),
+        costs=config_provider("costs"),
+        snapshots=config_provider("snapshots"),
+        scenario=config_provider("scenario"),
         RDIR=RDIR,
     input:
         networks=expand(
             RESULTS
             + "postnetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-            **config["scenario"]
+            **config["scenario"],
+            allow_missing=True,
         ),
-        costs="data/costs_{}.csv".format(config["costs"]["year"])
-        if config["foresight"] == "overnight"
-        else "data/costs_{}.csv".format(config["scenario"]["planning_horizons"][0]),
-        plots=expand(
+        costs=lambda w: (
+            resources("costs_{}.csv".format(config_provider("costs", "year")(w)))
+            if config_provider("foresight")(w) == "overnight"
+            else resources(
+                "costs_{}.csv".format(
+                    config_provider("scenario", "planning_horizons", 0)(w)
+                )
+            )
+        ),
+        ac_plot=expand(
+            resources("maps/power-network-s{simpl}-{clusters}.pdf"),
+            **config["scenario"],
+            allow_missing=True,
+        ),
+        costs_plot=expand(
             RESULTS
             + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf",
-            **config["scenario"]
+            **config["scenario"],
+            allow_missing=True,
+        ),
+        h2_plot=lambda w: expand(
+            (
+                RESULTS
+                + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf"
+                if config_provider("sector", "H2_network")(w)
+                else []
+            ),
+            **config["scenario"],
+            allow_missing=True,
+        ),
+        ch4_plot=lambda w: expand(
+            (
+                RESULTS
+                + "maps/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-ch4_network_{planning_horizons}.pdf"
+                if config_provider("sector", "gas_network")(w)
+                else []
+            ),
+            **config["scenario"],
+            allow_missing=True,
         ),
     output:
         nodal_costs=RESULTS + "csvs/nodal_costs.csv",
@@ -122,9 +225,7 @@ rule make_summary:
     resources:
         mem_mb=10000,
     log:
-        LOGS + "make_summary.log",
-    benchmark:
-        BENCHMARKS + "make_summary"
+        RESULTS + "logs/make_summary.log",
     conda:
         "../envs/environment.yaml"
     script:
@@ -133,12 +234,14 @@ rule make_summary:
 
 rule plot_summary:
     params:
-        countries=config["countries"],
-        planning_horizons=config["scenario"]["planning_horizons"],
-        sector_opts=config["scenario"]["sector_opts"],
-        emissions_scope=config["energy"]["emissions"],
-        eurostat_report_year=config["energy"]["eurostat_report_year"],
-        plotting=config["plotting"],
+        countries=config_provider("countries"),
+        planning_horizons=config_provider("scenario", "planning_horizons"),
+        emissions_scope=config_provider("energy", "emissions"),
+        eurostat_report_year=config_provider("energy", "eurostat_report_year"),
+        plotting=config_provider("plotting"),
+        foresight=config_provider("foresight"),
+        co2_budget=config_provider("co2_budget"),
+        sector=config_provider("sector"),
         RDIR=RDIR,
     input:
         costs=RESULTS + "csvs/costs.csv",
@@ -154,9 +257,7 @@ rule plot_summary:
     resources:
         mem_mb=10000,
     log:
-        LOGS + "plot_summary.log",
-    benchmark:
-        BENCHMARKS + "plot_summary"
+        RESULTS + "logs/plot_summary.log",
     conda:
         "../envs/environment.yaml"
     script:
@@ -178,7 +279,7 @@ STATISTICS_BARPLOTS = [
 
 rule plot_elec_statistics:
     params:
-        plotting=config["plotting"],
+        plotting=config_provider("plotting"),
         barplots=STATISTICS_BARPLOTS,
     input:
         network=RESULTS + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",

rules/retrieve.smk

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2023-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 
@@ -32,12 +32,12 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle",
         output:
             protected(expand("data/bundle/{file}", file=datafiles)),
         log:
-            LOGS + "retrieve_databundle.log",
+            "logs/retrieve_databundle.log",
         resources:
             mem_mb=1000,
         retries: 2
         conda:
-            "../envs/environment.yaml"
+            "../envs/retrieve.yaml"
         script:
             "../scripts/retrieve_databundle.py"
 
@@ -50,12 +50,12 @@ if config["enable"].get("retrieve_irena"):
             onwind="data/existing_infrastructure/onwind_capacity_IRENA.csv",
             solar="data/existing_infrastructure/solar_capacity_IRENA.csv",
         log:
-            LOGS + "retrieve_irena.log",
+            logs("retrieve_irena.log"),
         resources:
             mem_mb=1000,
         retries: 2
         conda:
-            "../envs/environment.yaml"
+            "../envs/retrieve.yaml"
         script:
             "../scripts/retrieve_irena.py"
 
@@ -83,23 +83,19 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True
 if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", True):
 
     rule retrieve_cost_data:
-        input:
-            HTTP.remote(
-                "raw.githubusercontent.com/PyPSA/technology-data/{}/outputs/".format(
-                    config["costs"]["version"]
-                )
-                + "costs_{year}.csv",
-                keep_local=True,
-            ),
+        params:
+            version=config_provider("costs", "version"),
         output:
-            "data/costs_{year}.csv",
+            resources("costs_{year}.csv"),
         log:
-            LOGS + "retrieve_cost_data_{year}.log",
+            logs("retrieve_cost_data_{year}.log"),
         resources:
             mem_mb=1000,
         retries: 2
-        run:
-            move(input[0], output[0])
+        conda:
+            "../envs/retrieve.yaml"
+        script:
+            "../scripts/retrieve_cost_data.py"
 
 
 if config["enable"]["retrieve"] and config["enable"].get(
@@ -114,9 +110,9 @@ if config["enable"]["retrieve"] and config["enable"].get(
                 static=True,
             ),
         output:
-            RESOURCES + "natura.tiff",
+            resources("natura.tiff"),
         log:
-            LOGS + "retrieve_natura_raster.log",
+            logs("retrieve_natura_raster.log"),
         resources:
             mem_mb=5000,
         retries: 2
@@ -155,17 +151,15 @@ if config["enable"]["retrieve"] and config["enable"].get(
             protected(expand("data/bundle-sector/{files}", files=datafiles)),
             *datafolders,
         log:
-            LOGS + "retrieve_sector_databundle.log",
+            "logs/retrieve_sector_databundle.log",
         retries: 2
         conda:
-            "../envs/environment.yaml"
+            "../envs/retrieve.yaml"
         script:
             "../scripts/retrieve_sector_databundle.py"
 
 
-if config["enable"]["retrieve"] and (
-    config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]
-):
+if config["enable"]["retrieve"]:
     datafiles = [
         "IGGIELGN_LNGs.geojson",
         "IGGIELGN_BorderPoints.geojson",
@@ -176,39 +170,31 @@ if config["enable"]["retrieve"] and (
 
     rule retrieve_gas_infrastructure_data:
         output:
-            protected(
-                expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles)
-            ),
+            expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles),
         log:
-            LOGS + "retrieve_gas_infrastructure_data.log",
+            logs("retrieve_gas_infrastructure_data.log"),
         retries: 2
         conda:
-            "../envs/environment.yaml"
+            "../envs/retrieve.yaml"
         script:
             "../scripts/retrieve_gas_infrastructure_data.py"
 
 if config["enable"]["retrieve"] and config["enable"].get("retrieve_opsd_load_data", True):
 
     rule retrieve_electricity_demand:
-        input:
-            HTTP.remote(
-                "data.open-power-system-data.org/time_series/{version}/time_series_60min_singleindex.csv".format(
-                version="2019-06-05"
-                    if config["snapshots"]["end"] < "2019"
-                    else "2020-10-06"
-                ),
-                keep_local=True,
-                static=True,
-            ),
+        params:
+            versions=["2019-06-05", "2020-10-06"],
         output:
-            RESOURCES + "load_raw.csv",
+            "data/electricity_demand_raw.csv",
         log:
-            LOGS + "retrieve_electricity_demand.log",
+            logs("retrieve_electricity_demand.log"),
         resources:
             mem_mb=5000,
         retries: 2
-        run:
-            move(input[0], output[0])
+        conda:
+            "../envs/retrieve.yaml"
+        script:
+            "../scripts/retrieve_electricity_demand.py"
 
 
 if config["enable"]["retrieve"] and config["enable"].get('retrieve_artificial_load_data', False):
@@ -216,7 +202,7 @@ if config["enable"]["retrieve"] and config["enable"].get('retrieve_artificial_lo
     rule retrieve_artificial_load_data:
         input: HTTP.remote("https://zenodo.org/record/7070438/files/demand_hourly.csv", keep_local=True, static=True)
         output: "data/load_artificial_raw.csv"
-        log: LOGS + "retrieve_artificial_load_data.log"
+        log: logs("retrieve_artificial_load_data.log")
         resources: mem_mb=5000,
         retries: 2
         run: move(input[0], output[0])
@@ -234,7 +220,7 @@ if config["enable"]["retrieve"]:
         output:
             protected("data/shipdensity_global.zip"),
         log:
-            LOGS + "retrieve_ship_raster.log",
+            logs("retrieve_ship_raster.log"),
         resources:
             mem_mb=5000,
         retries: 2
@@ -260,6 +246,22 @@ if config["enable"]["retrieve"]:
             validate_checksum(output[0], input[0])
 
 
+if config["enable"]["retrieve"]:
+
+    # Downloading LUISA Base Map for land cover and land use:
+    # Website: https://ec.europa.eu/jrc/en/luisa
+    rule retrieve_luisa_land_cover:
+        input:
+            HTTP.remote(
+                "jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/LUISA/EUROPE/Basemaps/LandUse/2018/LATEST/LUISA_basemap_020321_50m.tif",
+                static=True,
+            ),
+        output:
+            "data/LUISA_basemap_020321_50m.tif",
+        run:
+            move(input[0], output[0])
+
+
 if config["enable"]["retrieve"]:
     # Some logic to find the correct file URL
     # Sometimes files are released delayed or ahead of schedule, check which file is currently available
@@ -313,7 +315,7 @@ if config["enable"]["retrieve"]:
                 layer_path = (
                     f"/vsizip/{params.folder}/WDPA_{bYYYY}_Public_shp_{i}.zip"
                 )
-                print(f"Adding layer {i+1} of 3 to combined output file.")
+                print(f"Adding layer {i + 1} of 3 to combined output file.")
                 shell("ogr2ogr -f gpkg -update -append {output.gpkg} {layer_path}")
 
     rule download_wdpa_marine:
@@ -337,7 +339,7 @@ if config["enable"]["retrieve"]:
             for i in range(3):
                 # vsizip is special driver for directly working with zipped shapefiles in ogr2ogr
                 layer_path = f"/vsizip/{params.folder}/WDPA_WDOECM_{bYYYY}_Public_marine_shp_{i}.zip"
-                print(f"Adding layer {i+1} of 3 to combined output file.")
+                print(f"Adding layer {i + 1} of 3 to combined output file.")
                 shell("ogr2ogr -f gpkg -update -append {output.gpkg} {layer_path}")
 
 
@@ -354,7 +356,7 @@ if config["enable"]["retrieve"]:
         output:
             "data/validation/emission-spot-primary-market-auction-report-2019-data.xls",
         log:
-            LOGS + "retrieve_monthly_co2_prices.log",
+            logs("retrieve_monthly_co2_prices.log"),
         resources:
             mem_mb=5000,
         retries: 2
@@ -368,11 +370,11 @@ if config["enable"]["retrieve"]:
         output:
             "data/validation/energy-price-trends-xlsx-5619002.xlsx",
         log:
-            LOGS + "retrieve_monthly_fuel_prices.log",
+            logs("retrieve_monthly_fuel_prices.log"),
         resources:
             mem_mb=5000,
         retries: 2
         conda:
-            "../envs/environment.yaml"
+            "../envs/retrieve.yaml"
         script:
             "../scripts/retrieve_monthly_fuel_prices.py"

rules/solve_electricity.smk

@@ -1,39 +1,36 @@
-# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2023-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 
 
 rule solve_network:
     params:
-        solving=config["solving"],
-        foresight=config["foresight"],
-        planning_horizons=config["scenario"]["planning_horizons"],
-        co2_sequestration_potential=config["sector"].get(
-            "co2_sequestration_potential", 200
+        solving=config_provider("solving"),
+        foresight=config_provider("foresight"),
+        planning_horizons=config_provider("scenario", "planning_horizons"),
+        co2_sequestration_potential=config_provider(
+            "sector", "co2_sequestration_potential", default=200
         ),
+        custom_extra_functionality=input_custom_extra_functionality,
     input:
-        network=RESOURCES
-        + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
+        network=resources("networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"),
         config=RESULTS + "config.yaml",
     output:
         network=RESULTS
         + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
     log:
         solver=normpath(
-            LOGS
-            + "solve_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"
+            RESULTS
+            + "logs/solve_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"
         ),
-        python=LOGS
-        + "solve_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log",
+        python=RESULTS
+        + "logs/solve_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log",
     benchmark:
-        (
-            BENCHMARKS
-            + "solve_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}"
-        )
-    threads: 4
+        RESULTS + "benchmarks/solve_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}"
+    threads: solver_threads
     resources:
         mem_mb=memory,
-        walltime=config["solving"].get("walltime", "12:00:00"),
+        walltime=config_provider("solving", "walltime", default="12:00:00"),
     shadow:
         "minimal"
     conda:
@@ -44,7 +41,7 @@ rule solve_network:
 
 rule solve_operations_network:
     params:
-        options=config["solving"]["options"],
+        options=config_provider("solving", "options"),
     input:
         network=RESULTS
         + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
@@ -53,20 +50,20 @@ rule solve_operations_network:
         + "networks/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc",
     log:
         solver=normpath(
-            LOGS
-            + "solve_operations_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"
+            RESULTS
+            + "logs/solve_operations_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"
         ),
-        python=LOGS
-        + "solve_operations_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log",
+        python=RESULTS
+        + "logs/solve_operations_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log",
     benchmark:
         (
-            BENCHMARKS
-            + "solve_operations_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}"
+            RESULTS
+            + "benchmarks/solve_operations_network/elec{weather_year}_s{simpl}_{clusters}_ec_l{ll}_{opts}"
         )
     threads: 4
     resources:
         mem_mb=(lambda w: 10000 + 372 * int(w.clusters)),
-        walltime=config["solving"].get("walltime", "12:00:00"),
+        walltime=config_provider("solving", "walltime", default="12:00:00"),
     shadow:
         "minimal"
     conda:

rules/solve_myopic.smk

@@ -1,28 +1,31 @@
-# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2023-4 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 
 
 rule add_existing_baseyear:
     params:
-        baseyear=config["scenario"]["planning_horizons"][0],
-        sector=config["sector"],
-        existing_capacities=config["existing_capacities"],
-        costs=config["costs"],
+        baseyear=config_provider("scenario", "planning_horizons", 0),
+        sector=config_provider("sector"),
+        existing_capacities=config_provider("existing_capacities"),
+        costs=config_provider("costs"),
     input:
         network=RESULTS
         + "prenetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-        powerplants=RESOURCES + "powerplants.csv",
-        busmap_s=RESOURCES + "busmap_elec{weather_year}_s{simpl}.csv",
-        busmap=RESOURCES + "busmap_elec{weather_year}_s{simpl}_{clusters}.csv",
-        clustered_pop_layout=RESOURCES
-        + "pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv",
-        costs="data/costs_{}.csv".format(config["scenario"]["planning_horizons"][0]),
-        cop_soil_total=RESOURCES
-        + "cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc",
-        cop_air_total=RESOURCES
-        + "cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc",
-        existing_heating="data/existing_infrastructure/existing_heating_raw.csv",
+        powerplants=resources("powerplants.csv"),
+        busmap_s=resources("busmap_elec{weather_year}_s{simpl}.csv"),
+        busmap=resources("busmap_elec{weather_year}_s{simpl}_{clusters}.csv"),
+        clustered_pop_layout=resources("pop_layout_elec{weather_year}_s{simpl}_{clusters}.csv"),
+        costs=lambda w: resources(
+            "costs_{}.csv".format(
+                config_provider("scenario", "planning_horizons", 0)(w)
+            )
+        ),
+        cop_soil_total=resources("cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc"),
+        cop_air_total=resources("cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc"),
+        existing_heating_distribution=resources(
+            "existing_heating_distribution_elec{weather_year}_s{simpl}_{clusters}_{planning_horizons}.csv"
+        ),
         existing_solar="data/existing_infrastructure/solar_capacity_IRENA.csv",
         existing_onwind="data/existing_infrastructure/onwind_capacity_IRENA.csv",
         existing_offwind="data/existing_infrastructure/offwind_capacity_IRENA.csv",
@@ -30,17 +33,20 @@ rule add_existing_baseyear:
         RESULTS
         + "prenetworks-brownfield/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
     wildcard_constraints:
+        # TODO: The first planning_horizon needs to be aligned across scenarios
+        # snakemake does not support passing functions to wildcard_constraints
+        # reference: https://github.com/snakemake/snakemake/issues/2703
         planning_horizons=config["scenario"]["planning_horizons"][0],  #only applies to baseyear
     threads: 1
     resources:
         mem_mb=2000,
     log:
-        LOGS
-        + "add_existing_baseyear_elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
+        RESULTS
+        + "logs/add_existing_baseyear_elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
     benchmark:
         (
-            BENCHMARKS
-            + "add_existing_baseyear/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
+            RESULTS
+            + "benchmarks/add_existing_baseyear/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
         )
     conda:
         "../envs/environment.yaml"
@@ -48,20 +54,33 @@ rule add_existing_baseyear:
         "../scripts/add_existing_baseyear.py"
 
 
+def input_profile_tech_brownfield(w):
+    return {
+        f"profile_{tech}": resources(f"profile_{tech}.nc")
+        for tech in config_provider("electricity", "renewable_carriers")(w)
+        if tech != "hydro"
+    }
+
+
 rule add_brownfield:
     params:
-        H2_retrofit=config["sector"]["H2_retrofit"],
-        H2_retrofit_capacity_per_CH4=config["sector"]["H2_retrofit_capacity_per_CH4"],
-        threshold_capacity=config["existing_capacities"]["threshold_capacity"],
+        H2_retrofit=config_provider("sector", "H2_retrofit"),
+        H2_retrofit_capacity_per_CH4=config_provider(
+            "sector", "H2_retrofit_capacity_per_CH4"
+        ),
+        threshold_capacity=config_provider("existing_capacities", " threshold_capacity"),
+        snapshots=config_provider("snapshots"),
+        carriers=config_provider("electricity", "renewable_carriers"),
     input:
+        unpack(input_profile_tech_brownfield),
+        simplify_busmap=resources("busmap_elec{weather_year}_s{simpl}.csv"),
+        cluster_busmap=resources("busmap_elec{weather_year}_s{simpl}_{clusters}.csv"),
         network=RESULTS
         + "prenetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
         network_p=solved_previous_horizon,  #solved network at previous time step
-        costs="data/costs_{planning_horizons}.csv",
-        cop_soil_total=RESOURCES
-        + "cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc",
-        cop_air_total=RESOURCES
-        + "cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc",
+        costs=resources("costs_{planning_horizons}.csv"),
+        cop_soil_total=resources("cop_soil_total_elec{weather_year}_s{simpl}_{clusters}.nc"),
+        cop_air_total=resources("cop_air_total_elec{weather_year}_s{simpl}_{clusters}.nc"),
     output:
         RESULTS
         + "prenetworks-brownfield/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
@@ -69,12 +88,12 @@ rule add_brownfield:
     resources:
         mem_mb=10000,
     log:
-        LOGS
-        + "add_brownfield_elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
+        RESULTS
+        + "logs/add_brownfield_elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
     benchmark:
         (
-            BENCHMARKS
-            + "add_brownfield/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
+            RESULTS
+            + "benchmarks/add_brownfield/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
         )
     conda:
         "../envs/environment.yaml"
@@ -87,16 +106,17 @@ ruleorder: add_existing_baseyear > add_brownfield
 
 rule solve_sector_network_myopic:
     params:
-        solving=config["solving"],
-        foresight=config["foresight"],
-        planning_horizons=config["scenario"]["planning_horizons"],
-        co2_sequestration_potential=config["sector"].get(
-            "co2_sequestration_potential", 200
+        solving=config_provider("solving"),
+        foresight=config_provider("foresight"),
+        planning_horizons=config_provider("scenario", "planning_horizons"),
+        co2_sequestration_potential=config_provider(
+            "sector", "co2_sequestration_potential", default=200
         ),
+        custom_extra_functionality=input_custom_extra_functionality,
     input:
         network=RESULTS
         + "prenetworks-brownfield/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-        costs="data/costs_{planning_horizons}.csv",
+        costs=resources("costs_{planning_horizons}.csv"),
         config=RESULTS + "config.yaml",
     output:
         RESULTS
@@ -104,18 +124,18 @@ rule solve_sector_network_myopic:
     shadow:
         "shallow"
     log:
-        solver=LOGS
-        + "elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log",
-        python=LOGS
-        + "elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log",
-    threads: 4
+        solver=RESULTS
+        + "logs/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log",
+        python=RESULTS
+        + "logs/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log",
+    threads: solver_threads
     resources:
-        mem_mb=config["solving"]["mem"],
-        walltime=config["solving"].get("walltime", "12:00:00"),
+        mem_mb=config_provider("solving", "mem"),
+        walltime=config_provider("solving", "walltime", default="12:00:00"),
     benchmark:
         (
-            BENCHMARKS
-            + "solve_sector_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
+            RESULTS
+            + "benchmarks/solve_sector_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
         )
     conda:
         "../envs/environment.yaml"

rules/solve_overnight.smk

@@ -1,16 +1,17 @@
-# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2023-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 
 
 rule solve_sector_network:
     params:
-        solving=config["solving"],
-        foresight=config["foresight"],
-        planning_horizons=config["scenario"]["planning_horizons"],
-        co2_sequestration_potential=config["sector"].get(
-            "co2_sequestration_potential", 200
+        solving=config_provider("solving"),
+        foresight=config_provider("foresight"),
+        planning_horizons=config_provider("scenario", "planning_horizons"),
+        co2_sequestration_potential=config_provider(
+            "sector", "co2_sequestration_potential", default=200
         ),
+        custom_extra_functionality=input_custom_extra_functionality,
     input:
         network=RESULTS
         + "prenetworks/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
@@ -21,19 +22,20 @@ rule solve_sector_network:
     shadow:
         "shallow"
     log:
-        solver=LOGS
-        + "elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log",
-        python=LOGS
-        + "elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log",
-    threads: config["solving"]["solver"].get("threads", 4)
+        solver=RESULTS
+        + "logs/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log",
+        memory=RESULTS
+        + "logs/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_memory.log",
+        python=RESULTS
+        + "logs/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log",
+    threads: solver_threads
     resources:
-        mem_mb=config["solving"]["mem"],
-        walltime=config["solving"].get("walltime", "12:00:00"),
+        mem_mb=config_provider("solving", "mem"),
+        walltime=config_provider("solving", "walltime", default="12:00:00"),
     benchmark:
         (
             RESULTS
-            + BENCHMARKS
-            + "solve_sector_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
+            + "benchmarks/solve_sector_network/elec{weather_year}_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
         )
     conda:
         "../envs/environment.yaml"
@@ -43,7 +45,6 @@ rule solve_sector_network:
 
 rule solve_operations_network_other_year:
     input:
-        overrides="data/override_component_attrs",
         pre=RDIR
         + "/prenetworks/elec{weather_year}_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc",
         post=RDIR

rules/solve_perfect.smk

@@ -1,22 +1,29 @@
-# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2023-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 rule add_existing_baseyear:
     params:
-        baseyear=config["scenario"]["planning_horizons"][0],
-        sector=config["sector"],
-        existing_capacities=config["existing_capacities"],
-        costs=config["costs"],
+        baseyear=config_provider("scenario", "planning_horizons", 0),
+        sector=config_provider("sector"),
+        existing_capacities=config_provider("existing_capacities"),
+        costs=config_provider("costs"),
     input:
         network=RESULTS
         + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-        powerplants=RESOURCES + "powerplants.csv",
-        busmap_s=RESOURCES + "busmap_elec_s{simpl}.csv",
-        busmap=RESOURCES + "busmap_elec_s{simpl}_{clusters}.csv",
-        clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv",
-        costs="data/costs_{}.csv".format(config["scenario"]["planning_horizons"][0]),
-        cop_soil_total=RESOURCES + "cop_soil_total_elec_s{simpl}_{clusters}.nc",
-        cop_air_total=RESOURCES + "cop_air_total_elec_s{simpl}_{clusters}.nc",
+        powerplants=resources("powerplants.csv"),
+        busmap_s=resources("busmap_elec_s{simpl}.csv"),
+        busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"),
+        clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"),
+        costs=lambda w: resources(
+            "costs_{}.csv".format(
+                config_provider("scenario", "planning_horizons", 0)(w)
+            )
+        ),
+        cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"),
+        cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"),
+        existing_heating_distribution=resources(
+            "existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv"
+        ),
         existing_heating="data/existing_infrastructure/existing_heating_raw.csv",
         existing_solar="data/existing_infrastructure/solar_capacity_IRENA.csv",
         existing_onwind="data/existing_infrastructure/onwind_capacity_IRENA.csv",
@@ -30,12 +37,12 @@ rule add_existing_baseyear:
     resources:
         mem_mb=2000,
     log:
-        LOGS
-        + "add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
+        logs(
+            "add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"
+        ),
     benchmark:
-        (
-            BENCHMARKS
-            + "add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
+        benchmarks(
+            "add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
         )
     conda:
         "../envs/environment.yaml"
@@ -43,51 +50,28 @@ rule add_existing_baseyear:
         "../scripts/add_existing_baseyear.py"
 
 
-rule add_brownfield:
-    params:
-        H2_retrofit=config["sector"]["H2_retrofit"],
-        H2_retrofit_capacity_per_CH4=config["sector"]["H2_retrofit_capacity_per_CH4"],
-        threshold_capacity=config["existing_capacities"]["threshold_capacity"],
-    input:
-        network=RESULTS
-        + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-        network_p=solved_previous_horizon,  #solved network at previous time step
-        costs="data/costs_{planning_horizons}.csv",
-        cop_soil_total=RESOURCES + "cop_soil_total_elec_s{simpl}_{clusters}.nc",
-        cop_air_total=RESOURCES + "cop_air_total_elec_s{simpl}_{clusters}.nc",
-    output:
-        RESULTS
-        + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-    threads: 4
-    resources:
-        mem_mb=10000,
-    log:
-        LOGS
-        + "add_brownfield_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
-    benchmark:
-        (
-            BENCHMARKS
-            + "add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
-        )
-    conda:
-        "../envs/environment.yaml"
-    script:
-        "../scripts/add_brownfield.py"
+def input_network_year(w):
+    return {
+        f"network_{year}": RESULTS
+        + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}"
+        + f"_{year}.nc"
+        for year in config_provider("scenario", "planning_horizons")(w)[1:]
+    }
 
 
 rule prepare_perfect_foresight:
+    params:
+        costs=config_provider("costs"),
+        time_resolution=config_provider("clustering", "temporal", "sector"),
     input:
-        **{
-            f"network_{year}": RESULTS
-            + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_"
-            + f"{year}.nc"
-            for year in config["scenario"]["planning_horizons"][1:]
-        },
+        unpack(input_network_year),
         brownfield_network=lambda w: (
             RESULTS
             + "prenetworks-brownfield/"
             + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_"
-            + "{}.nc".format(str(config["scenario"]["planning_horizons"][0]))
+            + "{}.nc".format(
+                str(config_provider("scenario", "planning_horizons", 0)(w))
+            )
         ),
     output:
         RESULTS
@@ -96,12 +80,12 @@ rule prepare_perfect_foresight:
     resources:
         mem_mb=10000,
     log:
-        LOGS
-        + "prepare_perfect_foresight{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}.log",
+        logs(
+            "prepare_perfect_foresight{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}.log"
+        ),
     benchmark:
-        (
-            BENCHMARKS
-            + "prepare_perfect_foresight{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}"
+        benchmarks(
+            "prepare_perfect_foresight{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}"
         )
     conda:
         "../envs/environment.yaml"
@@ -111,24 +95,25 @@ rule prepare_perfect_foresight:
 
 rule solve_sector_network_perfect:
     params:
-        solving=config["solving"],
-        foresight=config["foresight"],
-        sector=config["sector"],
-        planning_horizons=config["scenario"]["planning_horizons"],
-        co2_sequestration_potential=config["sector"].get(
-            "co2_sequestration_potential", 200
+        solving=config_provider("solving"),
+        foresight=config_provider("foresight"),
+        sector=config_provider("sector"),
+        planning_horizons=config_provider("scenario", "planning_horizons"),
+        co2_sequestration_potential=config_provider(
+            "sector", "co2_sequestration_potential", default=200
         ),
+        custom_extra_functionality=input_custom_extra_functionality,
     input:
         network=RESULTS
         + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
-        costs="data/costs_2030.csv",
+        costs=resources("costs_2030.csv"),
         config=RESULTS + "config.yaml",
     output:
         RESULTS
         + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
-    threads: 4
+    threads: solver_threads
     resources:
-        mem_mb=config["solving"]["mem"],
+        mem_mb=config_provider("solving", "mem"),
     shadow:
         "shallow"
     log:
@@ -140,8 +125,8 @@ rule solve_sector_network_perfect:
         + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_memory.log",
     benchmark:
         (
-            BENCHMARKS
-            + "solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years}"
+            RESULTS
+            + "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years}"
         )
     conda:
         "../envs/environment.yaml"
@@ -149,18 +134,22 @@ rule solve_sector_network_perfect:
         "../scripts/solve_network.py"
 
 
+def input_networks_make_summary_perfect(w):
+    return {
+        f"networks_{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}": RESULTS
+        + f"postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc"
+        for simpl in config_provider("scenario", "simpl")(w)
+        for clusters in config_provider("scenario", "clusters")(w)
+        for opts in config_provider("scenario", "opts")(w)
+        for sector_opts in config_provider("scenario", "sector_opts")(w)
+        for ll in config_provider("scenario", "ll")(w)
+    }
+
+
 rule make_summary_perfect:
     input:
-        **{
-            f"networks_{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}": RESULTS
-            + f"postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc"
-            for simpl in config["scenario"]["simpl"]
-            for clusters in config["scenario"]["clusters"]
-            for opts in config["scenario"]["opts"]
-            for sector_opts in config["scenario"]["sector_opts"]
-            for ll in config["scenario"]["ll"]
-        },
-        costs="data/costs_2020.csv",
+        unpack(input_networks_make_summary_perfect),
+        costs=resources("costs_2020.csv"),
     output:
         nodal_costs=RESULTS + "csvs/nodal_costs.csv",
         nodal_capacities=RESULTS + "csvs/nodal_capacities.csv",
@@ -182,13 +171,10 @@ rule make_summary_perfect:
     resources:
         mem_mb=10000,
     log:
-        LOGS + "make_summary_perfect.log",
+        logs("make_summary_perfect.log"),
     benchmark:
-        (BENCHMARKS + "make_summary_perfect")
+        benchmarks("make_summary_perfect")
     conda:
         "../envs/environment.yaml"
     script:
         "../scripts/make_summary_perfect.py"
-
-
-ruleorder: add_existing_baseyear > add_brownfield

rules/validate.smk

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2023-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 
@@ -17,12 +17,12 @@ rule build_electricity_production:
     The data is used for validation of the optimization results.
     """
     params:
-        snapshots=config["snapshots"],
-        countries=config["countries"],
+        snapshots=config_provider("snapshots"),
+        countries=config_provider("countries"),
     output:
-        RESOURCES + "historical_electricity_production.csv",
+        resources("historical_electricity_production.csv"),
     log:
-        LOGS + "build_electricity_production.log",
+        logs("build_electricity_production.log"),
     resources:
         mem_mb=5000,
     script:
@@ -35,14 +35,14 @@ rule build_cross_border_flows:
     The data is used for validation of the optimization results.
     """
     params:
-        snapshots=config["snapshots"],
-        countries=config["countries"],
+        snapshots=config_provider("snapshots"),
+        countries=config_provider("countries"),
     input:
-        network=RESOURCES + "networks/base.nc",
+        network=resources("networks/base.nc"),
     output:
-        RESOURCES + "historical_cross_border_flows.csv",
+        resources("historical_cross_border_flows.csv"),
     log:
-        LOGS + "build_cross_border_flows.log",
+        logs("build_cross_border_flows.log"),
     resources:
         mem_mb=5000,
     script:
@@ -55,12 +55,12 @@ rule build_electricity_prices:
     The data is used for validation of the optimization results.
     """
     params:
-        snapshots=config["snapshots"],
-        countries=config["countries"],
+        snapshots=config_provider("snapshots"),
+        countries=config_provider("countries"),
     output:
-        RESOURCES + "historical_electricity_prices.csv",
+        resources("historical_electricity_prices.csv"),
     log:
-        LOGS + "build_electricity_prices.log",
+        logs("build_electricity_prices.log"),
     resources:
         mem_mb=5000,
     script:
@@ -70,7 +70,7 @@ rule build_electricity_prices:
 rule plot_validation_electricity_production:
     input:
         network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
-        electricity_production=RESOURCES + "historical_electricity_production.csv",
+        electricity_production=resources("historical_electricity_production.csv"),
     output:
         **{
             plot: RESULTS
@@ -85,10 +85,10 @@ rule plot_validation_electricity_production:
 
 rule plot_validation_cross_border_flows:
     params:
-        countries=config["countries"],
+        countries=config_provider("countries"),
     input:
         network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
-        cross_border_flows=RESOURCES + "historical_cross_border_flows.csv",
+        cross_border_flows=resources("historical_cross_border_flows.csv"),
     output:
         **{
             plot: RESULTS
@@ -104,7 +104,7 @@ rule plot_validation_cross_border_flows:
 rule plot_validation_electricity_prices:
     input:
         network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
-        electricity_prices=RESOURCES + "historical_electricity_prices.csv",
+        electricity_prices=resources("historical_electricity_prices.csv"),
     output:
         **{
             plot: RESULTS

scripts/__init__.py

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT

scripts/_benchmark.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2020-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 """
@@ -13,15 +13,15 @@ import os
 import sys
 import time
 
+from memory_profiler import _get_memory, choose_backend
+
 logger = logging.getLogger(__name__)
 
 # TODO: provide alternative when multiprocessing is not available
 try:
     from multiprocessing import Pipe, Process
 except ImportError:
-    from multiprocessing.dummy import Process, Pipe
-
-from memory_profiler import _get_memory, choose_backend
+    from multiprocessing.dummy import Pipe, Process
 
 
 # The memory logging facilities have been adapted from memory_profiler

scripts/_helpers.py

@@ -1,21 +1,23 @@
 # -*- coding: utf-8 -*-
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 
 import contextlib
+import copy
 import hashlib
 import logging
 import os
+import re
 import urllib
+from functools import partial
 from pathlib import Path
 
 import pandas as pd
 import pytz
 import requests
 import yaml
-from pypsa.components import component_attrs, components
-from pypsa.descriptors import Dict
+from snakemake.utils import update_config
 from tqdm import tqdm
 
 logger = logging.getLogger(__name__)
@@ -23,6 +25,106 @@ logger = logging.getLogger(__name__)
 REGION_COLS = ["geometry", "name", "x", "y", "country"]
 
 
+def get_run_path(fn, dir, rdir, shared_resources):
+    """
+    Dynamically provide paths based on shared resources and filename.
+
+    Use this function for snakemake rule inputs or outputs that should be
+    optionally shared across runs or created individually for each run.
+
+    Parameters
+    ----------
+    fn : str
+        The filename for the path to be generated.
+    dir : str
+        The base directory.
+    rdir : str
+        Relative directory for non-shared resources.
+    shared_resources : str or bool
+        Specifies which resources should be shared.
+        - If string is "base", special handling for shared "base" resources (see notes).
+        - If random string other than "base", this folder is used instead of the `rdir` keyword.
+        - If boolean, directly specifies if the resource is shared.
+
+    Returns
+    -------
+    str
+        Full path where the resource should be stored.
+
+    Notes
+    -----
+    Special case for "base" allows no wildcards other than "technology", "year"
+    and "scope" and excludes filenames starting with "networks/elec" or
+    "add_electricity". All other resources are shared.
+    """
+    if shared_resources == "base":
+        pattern = r"\{([^{}]+)\}"
+        existing_wildcards = set(re.findall(pattern, fn))
+        irrelevant_wildcards = {"technology", "year", "scope"}
+        no_relevant_wildcards = not existing_wildcards - irrelevant_wildcards
+        no_elec_rule = not fn.startswith("networks/elec") and not fn.startswith(
+            "add_electricity"
+        )
+        is_shared = no_relevant_wildcards and no_elec_rule
+    elif isinstance(shared_resources, str):
+        rdir = shared_resources + "/"
+        is_shared = True
+    elif isinstance(shared_resources, bool):
+        is_shared = shared_resources
+    else:
+        raise ValueError(
+            "shared_resources must be a boolean, str, or 'base' for special handling."
+        )
+
+    if is_shared:
+        return f"{dir}{fn}"
+    else:
+        return f"{dir}{rdir}{fn}"
+
+
+def path_provider(dir, rdir, shared_resources):
+    """
+    Returns a partial function that dynamically provides paths based on shared
+    resources and the filename.
+
+    Returns
+    -------
+    partial function
+        A partial function that takes a filename as input and
+        returns the path to the file based on the shared_resources parameter.
+    """
+    return partial(get_run_path, dir=dir, rdir=rdir, shared_resources=shared_resources)
+
+
+def get_opt(opts, expr, flags=None):
+    """
+    Return the first option matching the regular expression.
+
+    The regular expression is case-insensitive by default.
+    """
+    if flags is None:
+        flags = re.IGNORECASE
+    for o in opts:
+        match = re.match(expr, o, flags=flags)
+        if match:
+            return match.group(0)
+    return None
+
+
+def find_opt(opts, expr):
+    """
+    Return if available the float after the expression.
+    """
+    for o in opts:
+        if expr in o:
+            m = re.findall(r"m?\d+(?:[\.p]\d+)?", o)
+            if len(m) > 0:
+                return True, float(m[-1].replace("p", ".").replace("m", "-"))
+            else:
+                return True, None
+    return False, None
+
+
 # Define a context manager to temporarily mute print statements
 @contextlib.contextmanager
 def mute_print():
@@ -31,6 +133,21 @@ def mute_print():
             yield
 
 
+def set_scenario_config(snakemake):
+    scenario = snakemake.config["run"].get("scenarios", {})
+    if scenario.get("enable") and "run" in snakemake.wildcards.keys():
+        try:
+            with open(scenario["file"], "r") as f:
+                scenario_config = yaml.safe_load(f)
+        except FileNotFoundError:
+            # fallback for mock_snakemake
+            script_dir = Path(__file__).parent.resolve()
+            root_dir = script_dir.parent
+            with open(root_dir / scenario["file"], "r") as f:
+                scenario_config = yaml.safe_load(f)
+        update_config(snakemake.config, scenario_config[snakemake.wildcards.run])
+
+
 def configure_logging(snakemake, skip_handlers=False):
     """
     Configure the basic behaviour for the logging module.
@@ -50,6 +167,7 @@ def configure_logging(snakemake, skip_handlers=False):
         Do (not) skip the default handlers created for redirecting output to STDERR and file.
     """
     import logging
+    import sys
 
     kwargs = snakemake.config.get("logging", dict()).copy()
     kwargs.setdefault("level", "INFO")
@@ -73,6 +191,16 @@ def configure_logging(snakemake, skip_handlers=False):
         )
     logging.basicConfig(**kwargs)
 
+    # Setup a function to handle uncaught exceptions and include them with their stacktrace into logfiles
+    def handle_exception(exc_type, exc_value, exc_traceback):
+        # Log the exception
+        logger = logging.getLogger()
+        logger.error(
+            "Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback)
+        )
+
+    sys.excepthook = handle_exception
+
 
 def update_p_nom_max(n):
     # if extendable carriers (solar/onwind/...) have capacity >= 0,
@@ -193,7 +321,13 @@ def progress_retrieve(url, file, disable=False):
             urllib.request.urlretrieve(url, file, reporthook=update_to)
 
 
-def mock_snakemake(rulename, root_dir=None, configfiles=[], **wildcards):
+def mock_snakemake(
+    rulename,
+    root_dir=None,
+    configfiles=None,
+    submodule_dir="workflow/submodules/pypsa-eur",
+    **wildcards,
+):
     """
     This function is expected to be executed from the 'scripts'-directory of '
     the snakemake project. It returns a snakemake.script.Snakemake object,
@@ -209,6 +343,9 @@ def mock_snakemake(rulename, root_dir=None, configfiles=[], **wildcards):
         path to the root directory of the snakemake project
     configfiles: list, str
         list of configfiles to be used to update the config
+    submodule_dir: str, Path
+        in case PyPSA-Eur is used as a submodule, submodule_dir is
+        the path of pypsa-eur relative to the project directory.
     **wildcards:
         keyword arguments fixing the wildcards. Only necessary if wildcards are
         needed.
@@ -216,7 +353,6 @@ def mock_snakemake(rulename, root_dir=None, configfiles=[], **wildcards):
     import os
 
     import snakemake as sm
-    from packaging.version import Version, parse
     from pypsa.descriptors import Dict
     from snakemake.script import Snakemake
 
@@ -227,7 +363,10 @@ def mock_snakemake(rulename, root_dir=None, configfiles=[], **wildcards):
         root_dir = Path(root_dir).resolve()
 
     user_in_script_dir = Path.cwd().resolve() == script_dir
-    if user_in_script_dir:
+    if str(submodule_dir) in __file__:
+        # the submodule_dir path is only need to locate the project dir
+        os.chdir(Path(__file__[: __file__.find(str(submodule_dir))]))
+    elif user_in_script_dir:
         os.chdir(root_dir)
     elif Path.cwd().resolve() != root_dir:
         raise RuntimeError(
@@ -239,13 +378,14 @@ def mock_snakemake(rulename, root_dir=None, configfiles=[], **wildcards):
             if os.path.exists(p):
                 snakefile = p
                 break
-        kwargs = (
-            dict(rerun_triggers=[]) if parse(sm.__version__) > Version("7.7.0") else {}
-        )
-        if isinstance(configfiles, str):
+        if configfiles is None:
+            configfiles = []
+        elif isinstance(configfiles, str):
             configfiles = [configfiles]
 
-        workflow = sm.Workflow(snakefile, overwrite_configfiles=configfiles, **kwargs)
+        workflow = sm.Workflow(
+            snakefile, overwrite_configfiles=configfiles, rerun_triggers=[]
+        )
         workflow.include(snakefile)
 
         if configfiles:
@@ -262,7 +402,7 @@ def mock_snakemake(rulename, root_dir=None, configfiles=[], **wildcards):
 
         def make_accessable(*ios):
             for io in ios:
-                for i in range(len(io)):
+                for i, _ in enumerate(io):
                     io[i] = os.path.abspath(io[i])
 
         make_accessable(job.input, job.output, job.log)
@@ -309,17 +449,202 @@ def generate_periodic_profiles(dt_index, nodes, weekly_profile, localize=None):
     return week_df
 
 
-def parse(l):
-    return yaml.safe_load(l[0]) if len(l) == 1 else {l.pop(0): parse(l)}
+def parse(infix):
+    """
+    Recursively parse a chained wildcard expression into a dictionary or a YAML
+    object.
+
+    Parameters
+    ----------
+    list_to_parse : list
+        The list to parse.
+
+    Returns
+    -------
+    dict or YAML object
+        The parsed list.
+    """
+    if len(infix) == 1:
+        return yaml.safe_load(infix[0])
+    else:
+        return {infix.pop(0): parse(infix)}
 
 
-def update_config_with_sector_opts(config, sector_opts):
-    from snakemake.utils import update_config
+def update_config_from_wildcards(config, w, inplace=True):
+    """
+    Parses configuration settings from wildcards and updates the config.
+    """
 
-    for o in sector_opts.split("-"):
-        if o.startswith("CF+"):
-            l = o.split("+")[1:]
-            update_config(config, parse(l))
+    if not inplace:
+        config = copy.deepcopy(config)
+
+    if w.get("opts"):
+        opts = w.opts.split("-")
+
+        if nhours := get_opt(opts, r"^\d+(h|seg)$"):
+            config["clustering"]["temporal"]["resolution_elec"] = nhours
+
+        co2l_enable, co2l_value = find_opt(opts, "Co2L")
+        if co2l_enable:
+            config["electricity"]["co2limit_enable"] = True
+            if co2l_value is not None:
+                config["electricity"]["co2limit"] = (
+                    co2l_value * config["electricity"]["co2base"]
+                )
+
+        gasl_enable, gasl_value = find_opt(opts, "CH4L")
+        if gasl_enable:
+            config["electricity"]["gaslimit_enable"] = True
+            if gasl_value is not None:
+                config["electricity"]["gaslimit"] = gasl_value * 1e6
+
+        if "Ept" in opts:
+            config["costs"]["emission_prices"]["co2_monthly_prices"] = True
+
+        ep_enable, ep_value = find_opt(opts, "Ep")
+        if ep_enable:
+            config["costs"]["emission_prices"]["enable"] = True
+            if ep_value is not None:
+                config["costs"]["emission_prices"]["co2"] = ep_value
+
+        if "ATK" in opts:
+            config["autarky"]["enable"] = True
+            if "ATKc" in opts:
+                config["autarky"]["by_country"] = True
+
+        attr_lookup = {
+            "p": "p_nom_max",
+            "e": "e_nom_max",
+            "c": "capital_cost",
+            "m": "marginal_cost",
+        }
+        for o in opts:
+            flags = ["+e", "+p", "+m", "+c"]
+            if all(flag not in o for flag in flags):
+                continue
+            carrier, attr_factor = o.split("+")
+            attr = attr_lookup[attr_factor[0]]
+            factor = float(attr_factor[1:])
+            if not isinstance(config["adjustments"]["electricity"], dict):
+                config["adjustments"]["electricity"] = dict()
+            update_config(
+                config["adjustments"]["electricity"], {attr: {carrier: factor}}
+            )
+
+    if w.get("sector_opts"):
+        opts = w.sector_opts.split("-")
+
+        if "T" in opts:
+            config["sector"]["transport"] = True
+
+        if "H" in opts:
+            config["sector"]["heating"] = True
+
+        if "B" in opts:
+            config["sector"]["biomass"] = True
+
+        if "I" in opts:
+            config["sector"]["industry"] = True
+
+        if "A" in opts:
+            config["sector"]["agriculture"] = True
+
+        if "CCL" in opts:
+            config["solving"]["constraints"]["CCL"] = True
+
+        eq_value = get_opt(opts, r"^EQ+\d*\.?\d+(c|)")
+        for o in opts:
+            if eq_value is not None:
+                config["solving"]["constraints"]["EQ"] = eq_value
+            elif "EQ" in o:
+                config["solving"]["constraints"]["EQ"] = True
+            break
+
+        if "BAU" in opts:
+            config["solving"]["constraints"]["BAU"] = True
+
+        if "SAFE" in opts:
+            config["solving"]["constraints"]["SAFE"] = True
+
+        if nhours := get_opt(opts, r"^\d+(h|sn|seg)$"):
+            config["clustering"]["temporal"]["resolution_sector"] = nhours
+
+        if "decentral" in opts:
+            config["sector"]["electricity_transmission_grid"] = False
+
+        if "noH2network" in opts:
+            config["sector"]["H2_network"] = False
+
+        if "nowasteheat" in opts:
+            config["sector"]["use_fischer_tropsch_waste_heat"] = False
+            config["sector"]["use_methanolisation_waste_heat"] = False
+            config["sector"]["use_haber_bosch_waste_heat"] = False
+            config["sector"]["use_methanation_waste_heat"] = False
+            config["sector"]["use_fuel_cell_waste_heat"] = False
+            config["sector"]["use_electrolysis_waste_heat"] = False
+
+        if "nodistrict" in opts:
+            config["sector"]["district_heating"]["progress"] = 0.0
+
+        dg_enable, dg_factor = find_opt(opts, "dist")
+        if dg_enable:
+            config["sector"]["electricity_distribution_grid"] = True
+            if dg_factor is not None:
+                config["sector"][
+                    "electricity_distribution_grid_cost_factor"
+                ] = dg_factor
+
+        if "biomasstransport" in opts:
+            config["sector"]["biomass_transport"] = True
+
+        _, maxext = find_opt(opts, "linemaxext")
+        if maxext is not None:
+            config["lines"]["max_extension"] = maxext * 1e3
+            config["links"]["max_extension"] = maxext * 1e3
+
+        _, co2l_value = find_opt(opts, "Co2L")
+        if co2l_value is not None:
+            config["co2_budget"] = float(co2l_value)
+
+        if co2_distribution := get_opt(opts, r"^(cb)\d+(\.\d+)?(ex|be)$"):
+            config["co2_budget"] = co2_distribution
+
+        if co2_budget := get_opt(opts, r"^(cb)\d+(\.\d+)?$"):
+            config["co2_budget"] = float(co2_budget[2:])
+
+        attr_lookup = {
+            "p": "p_nom_max",
+            "e": "e_nom_max",
+            "c": "capital_cost",
+            "m": "marginal_cost",
+        }
+        for o in opts:
+            flags = ["+e", "+p", "+m", "+c"]
+            if all(flag not in o for flag in flags):
+                continue
+            carrier, attr_factor = o.split("+")
+            attr = attr_lookup[attr_factor[0]]
+            factor = float(attr_factor[1:])
+            if not isinstance(config["adjustments"]["sector"], dict):
+                config["adjustments"]["sector"] = dict()
+            update_config(config["adjustments"]["sector"], {attr: {carrier: factor}})
+
+        _, sdr_value = find_opt(opts, "sdr")
+        if sdr_value is not None:
+            config["costs"]["social_discountrate"] = sdr_value / 100
+
+        _, seq_limit = find_opt(opts, "seq")
+        if seq_limit is not None:
+            config["sector"]["co2_sequestration_potential"] = seq_limit
+
+        # any config option can be represented in wildcard
+        for o in opts:
+            if o.startswith("CF+"):
+                infix = o.split("+")[1:]
+                update_config(config, parse(infix))
+
+    if not inplace:
+        return config
 
 
 def get_checksum_from_zenodo(file_url):

scripts/add_brownfield.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2020-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 """
@@ -8,16 +8,20 @@ Prepares brownfield data from previous planning horizon.
 
 import logging
 
-logger = logging.getLogger(__name__)
-
-import pandas as pd
-
-idx = pd.IndexSlice
-
 import numpy as np
+import pandas as pd
 import pypsa
-from _helpers import update_config_with_sector_opts
+import xarray as xr
+from _helpers import (
+    configure_logging,
+    set_scenario_config,
+    update_config_from_wildcards,
+)
 from add_existing_baseyear import add_build_year_to_new_assets
+from pypsa.clustering.spatial import normed_or_uniform
+
+logger = logging.getLogger(__name__)
+idx = pd.IndexSlice
 
 
 def add_brownfield(n, n_p, year):
@@ -121,7 +125,7 @@ def add_brownfield(n, n_p, year):
 
 
 def disable_grid_expansion_if_LV_limit_hit(n):
-    if not "lv_limit" in n.global_constraints.index:
+    if "lv_limit" not in n.global_constraints.index:
         return
 
     total_expansion = (
@@ -133,9 +137,7 @@ def disable_grid_expansion_if_LV_limit_hit(n):
 
     # allow small numerical differences
     if lv_limit - total_expansion < 1:
-        logger.info(
-            f"LV is already reached (gap {diff} MWkm), disabling expansion and LV limit"
-        )
+        logger.info("LV is already reached, disabling expansion and LV limit")
         extendable_acs = n.lines.query("s_nom_extendable").index
         n.lines.loc[extendable_acs, "s_nom_extendable"] = False
         n.lines.loc[extendable_acs, "s_nom"] = n.lines.loc[extendable_acs, "s_nom_min"]
@@ -147,6 +149,57 @@ def disable_grid_expansion_if_LV_limit_hit(n):
         n.global_constraints.drop("lv_limit", inplace=True)
 
 
+def adjust_renewable_profiles(n, input_profiles, params, year):
+    """
+    Adjusts renewable profiles according to the renewable technology specified,
+    using the latest year below or equal to the selected year.
+    """
+
+    # spatial clustering
+    cluster_busmap = pd.read_csv(snakemake.input.cluster_busmap, index_col=0).squeeze()
+    simplify_busmap = pd.read_csv(
+        snakemake.input.simplify_busmap, index_col=0
+    ).squeeze()
+    clustermaps = simplify_busmap.map(cluster_busmap)
+    clustermaps.index = clustermaps.index.astype(str)
+
+    # temporal clustering
+    dr = pd.date_range(**params["snapshots"], freq="h")
+    snapshotmaps = (
+        pd.Series(dr, index=dr).where(lambda x: x.isin(n.snapshots), pd.NA).ffill()
+    )
+
+    for carrier in params["carriers"]:
+        if carrier == "hydro":
+            continue
+        with xr.open_dataset(getattr(input_profiles, "profile_" + carrier)) as ds:
+            if ds.indexes["bus"].empty or "year" not in ds.indexes:
+                continue
+
+            closest_year = max(
+                (y for y in ds.year.values if y <= year), default=min(ds.year.values)
+            )
+
+            p_max_pu = (
+                ds["profile"]
+                .sel(year=closest_year)
+                .transpose("time", "bus")
+                .to_pandas()
+            )
+
+            # spatial clustering
+            weight = ds["weight"].sel(year=closest_year).to_pandas()
+            weight = weight.groupby(clustermaps).transform(normed_or_uniform)
+            p_max_pu = (p_max_pu * weight).T.groupby(clustermaps).sum().T
+            p_max_pu.columns = p_max_pu.columns + f" {carrier}"
+
+            # temporal_clustering
+            p_max_pu = p_max_pu.groupby(snapshotmaps).mean()
+
+            # replace renewable time series
+            n.generators_t.p_max_pu.loc[:, p_max_pu.columns] = p_max_pu
+
+
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
@@ -158,13 +211,14 @@ if __name__ == "__main__":
             clusters="37",
             opts="",
             ll="v1.0",
-            sector_opts="168H-T-H-B-I-solar+p3-dist1",
+            sector_opts="168H-T-H-B-I-dist1",
             planning_horizons=2030,
         )
 
-    logging.basicConfig(level=snakemake.config["logging"]["level"])
+    configure_logging(snakemake)
+    set_scenario_config(snakemake)
 
-    update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts)
+    update_config_from_wildcards(snakemake.config, snakemake.wildcards)
 
     logger.info(f"Preparing brownfield from the file {snakemake.input.network_p}")
 
@@ -172,6 +226,8 @@ if __name__ == "__main__":
 
     n = pypsa.Network(snakemake.input.network)
 
+    adjust_renewable_profiles(n, snakemake.input, snakemake.params, year)
+
     add_build_year_to_new_assets(n, year)
 
     n_p = pypsa.Network(snakemake.input.network_p)

scripts/add_electricity.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 """
@@ -52,7 +52,7 @@ Inputs
         :scale: 34 %
 
 - ``data/geth2015_hydro_capacities.csv``: alternative to capacities above; not currently used!
-- ``resources/load.csv`` Hourly per-country load profiles.
+- ``resources/electricity_demand.csv`` Hourly per-country electricity demand profiles.
 - ``resources/regions_onshore.geojson``: confer :ref:`busregions`
 - ``resources/nuts3_shapes.geojson``: confer :ref:`shapes`
 - ``resources/powerplants.csv``: confer :ref:`powerplants`
@@ -93,7 +93,7 @@ import powerplantmatching as pm
 import pypsa
 import scipy.sparse as sparse
 import xarray as xr
-from _helpers import configure_logging, update_p_nom_max
+from _helpers import configure_logging, set_scenario_config, update_p_nom_max
 from powerplantmatching.export import map_country_bus
 from shapely.prepared import prep
 
@@ -178,6 +178,15 @@ def sanitize_carriers(n, config):
     n.carriers["color"] = n.carriers.color.where(n.carriers.color != "", colors)
 
 
+def sanitize_locations(n):
+    n.buses["x"] = n.buses.x.where(n.buses.x != 0, n.buses.location.map(n.buses.x))
+    n.buses["y"] = n.buses.y.where(n.buses.y != 0, n.buses.location.map(n.buses.y))
+    n.buses["country"] = n.buses.country.where(
+        n.buses.country.ne("") & n.buses.country.notnull(),
+        n.buses.location.map(n.buses.country),
+    )
+
+
 def add_co2_emissions(n, costs, carriers):
     """
     Add CO2 emissions to the network's carriers attribute.
@@ -288,16 +297,16 @@ def attach_load(n, regions, load, nuts3_shapes, ua_md_gdp, countries, scaling=1.
 
     ua_md_gdp = pd.read_csv(ua_md_gdp, dtype={"name": "str"}).set_index("name")
 
-    logger.info(f"Load data scaled with scalling factor {scaling}.")
+    logger.info(f"Load data scaled by factor {scaling}.")
     opsd_load *= scaling
 
     nuts3 = gpd.read_file(nuts3_shapes).set_index("index")
 
     def upsample(cntry, group):
-        l = opsd_load[cntry]
+        load = opsd_load[cntry]
 
         if len(group) == 1:
-            return pd.DataFrame({group.index[0]: l})
+            return pd.DataFrame({group.index[0]: load})
         nuts3_cntry = nuts3.loc[nuts3.country == cntry]
         transfer = shapes_to_shapes(group, nuts3_cntry.geometry).T.tocsr()
         gdp_n = pd.Series(
@@ -314,8 +323,8 @@ def attach_load(n, regions, load, nuts3_shapes, ua_md_gdp, countries, scaling=1.
             # overwrite factor because nuts3 provides no data for UA+MD
             factors = normed(ua_md_gdp.loc[group.index, "GDP_PPP"].squeeze())
         return pd.DataFrame(
-            factors.values * l.values[:, np.newaxis],
-            index=l.index,
+            factors.values * load.values[:, np.newaxis],
+            index=load.index,
             columns=factors.index,
         )
 
@@ -327,7 +336,9 @@ def attach_load(n, regions, load, nuts3_shapes, ua_md_gdp, countries, scaling=1.
         axis=1,
     )
 
-    n.madd("Load", substation_lv_i, bus=substation_lv_i, p_set=load)
+    n.madd(
+        "Load", substation_lv_i, bus=substation_lv_i, p_set=load
+    )  # carrier="electricity"
 
 
 def update_transmission_costs(n, costs, length_factor=1.0):
@@ -374,6 +385,10 @@ def attach_wind_and_solar(
             if ds.indexes["bus"].empty:
                 continue
 
+            # if-statement for compatibility with old profiles
+            if "year" in ds.indexes:
+                ds = ds.sel(year=ds.year.min(), drop=True)
+
             supcar = car.split("-", 2)[0]
             if supcar == "offwind":
                 underwater_fraction = ds["underwater_fraction"].to_pandas()
@@ -504,8 +519,8 @@ def attach_conventional_generators(
                     snakemake.input[f"conventional_{carrier}_{attr}"], index_col=0
                 ).iloc[:, 0]
                 bus_values = n.buses.country.map(values)
-                n.generators[attr].update(
-                    n.generators.loc[idx].bus.map(bus_values).dropna()
+                n.generators.update(
+                    {attr: n.generators.loc[idx].bus.map(bus_values).dropna()}
                 )
             else:
                 # Single value affecting all generators of technology k indiscriminantely of country
@@ -622,7 +637,7 @@ def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **par
             hydro.max_hours > 0, hydro.country.map(max_hours_country)
         ).fillna(6)
 
-        if flatten_dispatch := params.get("flatten_dispatch", False):
+        if params.get("flatten_dispatch", False):
             buffer = params.get("flatten_dispatch_buffer", 0.2)
             average_capacity_factor = inflow_t[hydro.index].mean() / hydro["p_nom"]
             p_max_pu = (average_capacity_factor + buffer).clip(upper=1)
@@ -647,77 +662,6 @@ def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **par
         )
 
 
-def attach_extendable_generators(n, costs, ppl, carriers):
-    logger.warning(
-        "The function `attach_extendable_generators` is deprecated in v0.5.0."
-    )
-    add_missing_carriers(n, carriers)
-    add_co2_emissions(n, costs, carriers)
-
-    for tech in carriers:
-        if tech.startswith("OCGT"):
-            ocgt = (
-                ppl.query("carrier in ['OCGT', 'CCGT']")
-                .groupby("bus", as_index=False)
-                .first()
-            )
-            n.madd(
-                "Generator",
-                ocgt.index,
-                suffix=" OCGT",
-                bus=ocgt["bus"],
-                carrier=tech,
-                p_nom_extendable=True,
-                p_nom=0.0,
-                capital_cost=costs.at["OCGT", "capital_cost"],
-                marginal_cost=costs.at["OCGT", "marginal_cost"],
-                efficiency=costs.at["OCGT", "efficiency"],
-            )
-
-        elif tech.startswith("CCGT"):
-            ccgt = (
-                ppl.query("carrier in ['OCGT', 'CCGT']")
-                .groupby("bus", as_index=False)
-                .first()
-            )
-            n.madd(
-                "Generator",
-                ccgt.index,
-                suffix=" CCGT",
-                bus=ccgt["bus"],
-                carrier=tech,
-                p_nom_extendable=True,
-                p_nom=0.0,
-                capital_cost=costs.at["CCGT", "capital_cost"],
-                marginal_cost=costs.at["CCGT", "marginal_cost"],
-                efficiency=costs.at["CCGT", "efficiency"],
-            )
-
-        elif tech.startswith("nuclear"):
-            nuclear = (
-                ppl.query("carrier == 'nuclear'").groupby("bus", as_index=False).first()
-            )
-            n.madd(
-                "Generator",
-                nuclear.index,
-                suffix=" nuclear",
-                bus=nuclear["bus"],
-                carrier=tech,
-                p_nom_extendable=True,
-                p_nom=0.0,
-                capital_cost=costs.at["nuclear", "capital_cost"],
-                marginal_cost=costs.at["nuclear", "marginal_cost"],
-                efficiency=costs.at["nuclear", "efficiency"],
-            )
-
-        else:
-            raise NotImplementedError(
-                "Adding extendable generators for carrier "
-                "'{tech}' is not implemented, yet. "
-                "Only OCGT, CCGT and nuclear are allowed at the moment."
-            )
-
-
 def attach_OPSD_renewables(n: pypsa.Network, tech_map: Dict[str, List[str]]) -> None:
     """
     Attach renewable capacities from the OPSD dataset to the network.
@@ -749,8 +693,8 @@ def attach_OPSD_renewables(n: pypsa.Network, tech_map: Dict[str, List[str]]) ->
         caps = caps.groupby(["bus"]).Capacity.sum()
         caps = caps / gens_per_bus.reindex(caps.index, fill_value=1)
 
-        n.generators.p_nom.update(gens.bus.map(caps).dropna())
-        n.generators.p_nom_min.update(gens.bus.map(caps).dropna())
+        n.generators.update({"p_nom": gens.bus.map(caps).dropna()})
+        n.generators.update({"p_nom_min": gens.bus.map(caps).dropna()})
 
 
 def estimate_renewable_capacities(
@@ -855,6 +799,7 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("add_electricity", weather_year="")
     configure_logging(snakemake)
+    set_scenario_config(snakemake)
 
     params = snakemake.params
 
@@ -978,7 +923,7 @@ if __name__ == "__main__":
 
     sanitize_carriers(n, snakemake.config)
 
-    if snakemake.config["enable"].get("drop_leap_days", True):
+    if snakemake.params.drop_leap_day:
         drop_leap_day(n)
 
     n.meta = snakemake.config

scripts/add_existing_baseyear.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2020-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 """
@@ -8,25 +8,24 @@ horizon.
 """
 
 import logging
-
-logger = logging.getLogger(__name__)
-
-import pandas as pd
-
-idx = pd.IndexSlice
-
 from types import SimpleNamespace
 
 import country_converter as coco
 import numpy as np
+import pandas as pd
 import pypsa
 import xarray as xr
-from _helpers import update_config_with_sector_opts
+from _helpers import (
+    configure_logging,
+    set_scenario_config,
+    update_config_from_wildcards,
+)
 from add_electricity import sanitize_carriers
 from prepare_sector_network import cluster_heat_buses, define_spatial, prepare_costs
 
+logger = logging.getLogger(__name__)
 cc = coco.CountryConverter()
-
+idx = pd.IndexSlice
 spatial = SimpleNamespace()
 
 
@@ -53,7 +52,7 @@ def add_build_year_to_new_assets(n, baseyear):
             "series"
         ) & n.component_attrs[c.name].status.str.contains("Input")
         for attr in n.component_attrs[c.name].index[selection]:
-            c.pnl[attr].rename(columns=rename, inplace=True)
+            c.pnl[attr] = c.pnl[attr].rename(columns=rename)
 
 
 def add_existing_renewables(df_agg):
@@ -305,7 +304,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas
         else:
             bus0 = vars(spatial)[carrier[generator]].nodes
             if "EU" not in vars(spatial)[carrier[generator]].locations:
-                bus0 = bus0.intersection(capacity.index + " gas")
+                bus0 = bus0.intersection(capacity.index + " " + carrier[generator])
 
             # check for missing bus
             missing_bus = pd.Index(bus0).difference(n.buses.index)
@@ -407,104 +406,23 @@ def add_heating_capacities_installed_before_baseyear(
     """
     logger.debug(f"Adding heating capacities installed before {baseyear}")
 
-    # Add existing heating capacities, data comes from the study
-    # "Mapping and analyses of the current and future (2020 - 2030)
-    # heating/cooling fuel deployment (fossil/renewables) "
-    # https://ec.europa.eu/energy/studies/mapping-and-analyses-current-and-future-2020-2030-heatingcooling-fuel-deployment_en?redir=1
-    # file: "WP2_DataAnnex_1_BuildingTechs_ForPublication_201603.xls" -> "existing_heating_raw.csv".
-    # TODO start from original file
-
-    # retrieve existing heating capacities
-    techs = [
-        "gas boiler",
-        "oil boiler",
-        "resistive heater",
-        "air heat pump",
-        "ground heat pump",
-    ]
-    df = pd.read_csv(snakemake.input.existing_heating, index_col=0, header=0)
-
-    # data for Albania, Montenegro and Macedonia not included in database
-    df.loc["Albania"] = np.nan
-    df.loc["Montenegro"] = np.nan
-    df.loc["Macedonia"] = np.nan
-
-    df.fillna(0.0, inplace=True)
-
-    # convert GW to MW
-    df *= 1e3
-
-    df.index = cc.convert(df.index, to="iso2")
-
-    # coal and oil boilers are assimilated to oil boilers
-    df["oil boiler"] = df["oil boiler"] + df["coal boiler"]
-    df.drop(["coal boiler"], axis=1, inplace=True)
-
-    # distribute technologies to nodes by population
-    pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0)
-
-    nodal_df = df.loc[pop_layout.ct]
-    nodal_df.index = pop_layout.index
-    nodal_df = nodal_df.multiply(pop_layout.fraction, axis=0)
-
-    # split existing capacities between residential and services
-    # proportional to energy demand
-    p_set_sum = n.loads_t.p_set.sum()
-    ratio_residential = pd.Series(
-        [
-            (
-                p_set_sum[f"{node} residential rural heat"]
-                / (
-                    p_set_sum[f"{node} residential rural heat"]
-                    + p_set_sum[f"{node} services rural heat"]
-                )
-            )
-            # if rural heating demand for one of the nodes doesn't exist,
-            # then columns were dropped before and heating demand share should be 0.0
-            if all(
-                f"{node} {service} rural heat" in p_set_sum.index
-                for service in ["residential", "services"]
-            )
-            else 0.0
-            for node in nodal_df.index
-        ],
-        index=nodal_df.index,
+    existing_heating = pd.read_csv(
+        snakemake.input.existing_heating_distribution, header=[0, 1], index_col=0
     )
 
-    for tech in techs:
-        nodal_df["residential " + tech] = nodal_df[tech] * ratio_residential
-        nodal_df["services " + tech] = nodal_df[tech] * (1 - ratio_residential)
+    techs = existing_heating.columns.get_level_values(1).unique()
 
-    names = [
-        "residential rural",
-        "services rural",
-        "residential urban decentral",
-        "services urban decentral",
-        "urban central",
-    ]
-
-    nodes = {}
-    p_nom = {}
-    for name in names:
+    for name in existing_heating.columns.get_level_values(0).unique():
         name_type = "central" if name == "urban central" else "decentral"
-        nodes[name] = pd.Index(
-            [
-                n.buses.at[index, "location"]
-                for index in n.buses.index[
-                    n.buses.index.str.contains(name)
-                    & n.buses.index.str.contains("heat")
-                ]
-            ]
-        )
-        heat_pump_type = "air" if "urban" in name else "ground"
-        heat_type = "residential" if "residential" in name else "services"
 
-        if name == "urban central":
-            p_nom[name] = nodal_df["air heat pump"][nodes[name]]
+        nodes = pd.Index(n.buses.location[n.buses.index.str.contains(f"{name} heat")])
+
+        if (name_type != "central") and options["electricity_distribution_grid"]:
+            nodes_elec = nodes + " low voltage"
         else:
-            p_nom[name] = nodal_df[f"{heat_type} {heat_pump_type} heat pump"][
-                nodes[name]
-            ]
+            nodes_elec = nodes
+
+        heat_pump_type = "air" if "urban" in name else "ground"
 
         # Add heat pumps
         costs_name = f"decentral {heat_pump_type}-sourced heat pump"
@@ -512,7 +430,7 @@ def add_heating_capacities_installed_before_baseyear(
         cop = {"air": ashp_cop, "ground": gshp_cop}
 
         if time_dep_hp_cop:
-            efficiency = cop[heat_pump_type][nodes[name]]
+            efficiency = cop[heat_pump_type][nodes]
         else:
             efficiency = costs.at[costs_name, "efficiency"]
 
@@ -520,32 +438,33 @@ def add_heating_capacities_installed_before_baseyear(
             if int(grouping_year) + default_lifetime <= int(baseyear):
                 continue
 
-            # installation is assumed to be linear for the past 25 years (default lifetime)
+            # installation is assumed to be linear for the past default_lifetime years
             ratio = (int(grouping_year) - int(grouping_years[i - 1])) / default_lifetime
 
             n.madd(
                 "Link",
-                nodes[name],
+                nodes,
                 suffix=f" {name} {heat_pump_type} heat pump-{grouping_year}",
-                bus0=nodes[name],
-                bus1=nodes[name] + " " + name + " heat",
+                bus0=nodes_elec,
+                bus1=nodes + " " + name + " heat",
                 carrier=f"{name} {heat_pump_type} heat pump",
                 efficiency=efficiency,
                 capital_cost=costs.at[costs_name, "efficiency"]
                 * costs.at[costs_name, "fixed"],
-                p_nom=p_nom[name] * ratio / costs.at[costs_name, "efficiency"],
+                p_nom=existing_heating.loc[nodes, (name, f"{heat_pump_type} heat pump")]
+                * ratio
+                / costs.at[costs_name, "efficiency"],
                 build_year=int(grouping_year),
                 lifetime=costs.at[costs_name, "lifetime"],
             )
 
             # add resistive heater, gas boilers and oil boilers
-            # (50% capacities to rural buses, 50% to urban buses)
             n.madd(
                 "Link",
-                nodes[name],
+                nodes,
                 suffix=f" {name} resistive heater-{grouping_year}",
-                bus0=nodes[name],
-                bus1=nodes[name] + " " + name + " heat",
+                bus0=nodes_elec,
+                bus1=nodes + " " + name + " heat",
                 carrier=name + " resistive heater",
                 efficiency=costs.at[f"{name_type} resistive heater", "efficiency"],
                 capital_cost=(
@@ -553,21 +472,20 @@ def add_heating_capacities_installed_before_baseyear(
                     * costs.at[f"{name_type} resistive heater", "fixed"]
                 ),
                 p_nom=(
-                    0.5
-                    * nodal_df[f"{heat_type} resistive heater"][nodes[name]]
+                    existing_heating.loc[nodes, (name, "resistive heater")]
                     * ratio
                     / costs.at[f"{name_type} resistive heater", "efficiency"]
                 ),
                 build_year=int(grouping_year),
-                lifetime=costs.at[costs_name, "lifetime"],
+                lifetime=costs.at[f"{name_type} resistive heater", "lifetime"],
             )
 
             n.madd(
                 "Link",
-                nodes[name],
+                nodes,
                 suffix=f" {name} gas boiler-{grouping_year}",
-                bus0=spatial.gas.nodes,
-                bus1=nodes[name] + " " + name + " heat",
+                bus0="EU gas" if "EU gas" in spatial.gas.nodes else nodes + " gas",
+                bus1=nodes + " " + name + " heat",
                 bus2="co2 atmosphere",
                 carrier=name + " gas boiler",
                 efficiency=costs.at[f"{name_type} gas boiler", "efficiency"],
@@ -577,8 +495,7 @@ def add_heating_capacities_installed_before_baseyear(
                     * costs.at[f"{name_type} gas boiler", "fixed"]
                 ),
                 p_nom=(
-                    0.5
-                    * nodal_df[f"{heat_type} gas boiler"][nodes[name]]
+                    existing_heating.loc[nodes, (name, "gas boiler")]
                     * ratio
                     / costs.at[f"{name_type} gas boiler", "efficiency"]
                 ),
@@ -588,20 +505,21 @@ def add_heating_capacities_installed_before_baseyear(
 
             n.madd(
                 "Link",
-                nodes[name],
+                nodes,
                 suffix=f" {name} oil boiler-{grouping_year}",
                 bus0=spatial.oil.nodes,
-                bus1=nodes[name] + " " + name + " heat",
+                bus1=nodes + " " + name + " heat",
                 bus2="co2 atmosphere",
                 carrier=name + " oil boiler",
                 efficiency=costs.at["decentral oil boiler", "efficiency"],
                 efficiency2=costs.at["oil", "CO2 intensity"],
                 capital_cost=costs.at["decentral oil boiler", "efficiency"]
                 * costs.at["decentral oil boiler", "fixed"],
-                p_nom=0.5
-                * nodal_df[f"{heat_type} oil boiler"][nodes[name]]
-                * ratio
-                / costs.at["decentral oil boiler", "efficiency"],
+                p_nom=(
+                    existing_heating.loc[nodes, (name, "oil boiler")]
+                    * ratio
+                    / costs.at["decentral oil boiler", "efficiency"]
+                ),
                 build_year=int(grouping_year),
                 lifetime=costs.at[f"{name_type} gas boiler", "lifetime"],
             )
@@ -627,11 +545,8 @@ def add_heating_capacities_installed_before_baseyear(
                 ],
             )
 
-            # drop assets which are at the end of their lifetime
-            links_i = n.links[(n.links.build_year + n.links.lifetime <= baseyear)].index
-            n.mremove("Link", links_i)
-
 
+# %%
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
@@ -644,16 +559,16 @@ if __name__ == "__main__":
             clusters="37",
             ll="v1.0",
             opts="",
-            sector_opts="1p7-4380H-T-H-B-I-A-solar+p3-dist1",
+            sector_opts="8760-T-H-B-I-A-dist1",
             planning_horizons=2020,
         )
 
-    logging.basicConfig(level=snakemake.config["logging"]["level"])
+    configure_logging(snakemake)
+    set_scenario_config(snakemake)
 
-    update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts)
+    update_config_from_wildcards(snakemake.config, snakemake.wildcards)
 
     options = snakemake.params.sector
-    opts = snakemake.wildcards.sector_opts.split("-")
 
     baseyear = snakemake.params.baseyear
 
@@ -676,7 +591,7 @@ if __name__ == "__main__":
         n, grouping_years_power, costs, baseyear
     )
 
-    if "H" in opts:
+    if options["heating"]:
         time_dep_hp_cop = options["time_dep_hp_cop"]
         ashp_cop = (
             xr.open_dataarray(snakemake.input.cop_air_total)
@@ -688,7 +603,9 @@ if __name__ == "__main__":
             .to_pandas()
             .reindex(index=n.snapshots)
         )
-        default_lifetime = snakemake.params.costs["fill_values"]["lifetime"]
+        default_lifetime = snakemake.params.existing_capacities[
+            "default_heating_lifetime"
+        ]
         add_heating_capacities_installed_before_baseyear(
             n,
             baseyear,

scripts/add_extra_components.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 
@@ -55,8 +55,8 @@ import logging
 import numpy as np
 import pandas as pd
 import pypsa
-from _helpers import configure_logging
-from add_electricity import load_costs, sanitize_carriers
+from _helpers import configure_logging, set_scenario_config
+from add_electricity import load_costs, sanitize_carriers, sanitize_locations
 
 idx = pd.IndexSlice
 
@@ -100,10 +100,9 @@ def attach_stores(n, costs, extendable_carriers):
     n.madd("Carrier", carriers)
 
     buses_i = n.buses.index
-    bus_sub_dict = {k: n.buses[k].values for k in ["x", "y", "country"]}
 
     if "H2" in carriers:
-        h2_buses_i = n.madd("Bus", buses_i + " H2", carrier="H2", **bus_sub_dict)
+        h2_buses_i = n.madd("Bus", buses_i + " H2", carrier="H2", location=buses_i)
 
         n.madd(
             "Store",
@@ -143,7 +142,7 @@ def attach_stores(n, costs, extendable_carriers):
 
     if "battery" in carriers:
         b_buses_i = n.madd(
-            "Bus", buses_i + " battery", carrier="battery", **bus_sub_dict
+            "Bus", buses_i + " battery", carrier="battery", location=buses_i
         )
 
         n.madd(
@@ -233,6 +232,7 @@ if __name__ == "__main__":
             "add_extra_components", weather_year="", simpl="", clusters=5
         )
     configure_logging(snakemake)
+    set_scenario_config(snakemake)
 
     n = pypsa.Network(snakemake.input.network)
     extendable_carriers = snakemake.params.extendable_carriers
@@ -248,6 +248,7 @@ if __name__ == "__main__":
     attach_hydrogen_pipelines(n, costs, extendable_carriers)
 
     sanitize_carriers(n, snakemake.config)
+    sanitize_locations(n)
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
     n.export_to_netcdf(snakemake.output[0])

scripts/base_network.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 
@@ -75,11 +75,14 @@ import shapely
 import shapely.prepared
 import shapely.wkt
 import yaml
-from _helpers import configure_logging
+from _helpers import configure_logging, set_scenario_config
+from packaging.version import Version, parse
 from scipy import spatial
 from scipy.sparse import csgraph
 from shapely.geometry import LineString, Point
 
+PD_GE_2_2 = parse(pd.__version__) >= Version("2.2")
+
 logger = logging.getLogger(__name__)
 
 
@@ -136,7 +139,9 @@ def _load_buses_from_eg(eg_buses, europe_shape, config_elec):
     )
 
     buses["carrier"] = buses.pop("dc").map({True: "DC", False: "AC"})
-    buses["under_construction"] = buses["under_construction"].fillna(False).astype(bool)
+    buses["under_construction"] = buses.under_construction.where(
+        lambda s: s.notnull(), False
+    ).astype(bool)
 
     # remove all buses outside of all countries including exclusive economic zones (offshore)
     europe_shape = gpd.read_file(europe_shape).loc[0, "geometry"]
@@ -520,12 +525,13 @@ def _set_countries_and_substations(n, config, country_shapes, offshore_shapes):
         )
         return pd.Series(key, index)
 
+    compat_kws = dict(include_groups=False) if PD_GE_2_2 else {}
     gb = buses.loc[substation_b].groupby(
         ["x", "y"], as_index=False, group_keys=False, sort=False
     )
-    bus_map_low = gb.apply(prefer_voltage, "min")
+    bus_map_low = gb.apply(prefer_voltage, "min", **compat_kws)
     lv_b = (bus_map_low == bus_map_low.index).reindex(buses.index, fill_value=False)
-    bus_map_high = gb.apply(prefer_voltage, "max")
+    bus_map_high = gb.apply(prefer_voltage, "max", **compat_kws)
     hv_b = (bus_map_high == bus_map_high.index).reindex(buses.index, fill_value=False)
 
     onshore_b = pd.Series(False, buses.index)
@@ -551,6 +557,7 @@ def _set_countries_and_substations(n, config, country_shapes, offshore_shapes):
     for b, df in product(("bus0", "bus1"), (n.lines, n.links)):
         has_connections_b |= ~df.groupby(b).under_construction.min()
 
+    buses["onshore_bus"] = onshore_b
     buses["substation_lv"] = (
         lv_b & onshore_b & (~buses["under_construction"]) & has_connections_b
     )
@@ -723,11 +730,12 @@ def base_network(
     transformers = _set_electrical_parameters_transformers(transformers, config)
     links = _set_electrical_parameters_links(links, config, links_p_nom)
     converters = _set_electrical_parameters_converters(converters, config)
+    snapshots = snakemake.params.snapshots
 
     n = pypsa.Network()
     n.name = "PyPSA-Eur"
 
-    n.set_snapshots(pd.date_range(freq="h", **config["snapshots"]))
+    n.set_snapshots(pd.date_range(freq="h", **snapshots))
     n.madd("Carrier", ["AC", "DC"])
 
     n.import_components_from_dataframe(buses, "Bus")
@@ -759,6 +767,7 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("base_network")
     configure_logging(snakemake)
+    set_scenario_config(snakemake)
 
     n = base_network(
         snakemake.input.eg_buses,