Merged https://github.com/pypsa/pypsa-eur

2024-03-14 12:03:56 +01:00 · 2024-03-14 12:03:56 +01:00 · 09765ad060
commit 09765ad060
parent 025890fc59 0a0a35e4a4
184 changed files with 8931 additions and 5068 deletions
--- a/.git-blame-ignore-revs
+++ b/.git-blame-ignore-revs
@ -6,3 +6,4 @@
 5d1ef8a64055a039aa4a0834d2d26fe7752fe9a0
 92080b1cd2ca5f123158571481722767b99c2b27
 13769f90af4500948b0376d57df4cceaa13e78b5
+9865a970893d9e515786f33c629b14f71645bf1e
--- a/.gitattributes
+++ b/.gitattributes
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0

--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2021-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0

@ -19,7 +19,7 @@ on:
  - cron: "0 5 * * TUE"

 env:
-  DATA_CACHE_NUMBER: 2
+  DATA_CACHE_NUMBER: 1

 jobs:
  build:
@ -32,7 +32,14 @@ jobs:
        - ubuntu-latest
        - macos-latest
        - windows-latest
-
+        inhouse:
+        - stable
+        - master
+        exclude:
+        - os: macos-latest
+          inhouse: master
+        - os: windows-latest
+          inhouse: master
    runs-on: ${{ matrix.os }}

    defaults:
@ -46,16 +53,6 @@ jobs:
      run: |
        echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc

-    - name: Add solver to environment
-      run: |
-        echo -e "- glpk\n- ipopt<3.13.3" >> envs/environment.yaml
-      if: ${{ matrix.os }} == 'windows-latest'
-
-    - name: Add solver to environment
-      run: |
-        echo -e "- glpk\n- ipopt" >> envs/environment.yaml
-      if: ${{ matrix.os }} != 'windows-latest'
-
    - name: Setup micromamba
      uses: mamba-org/setup-micromamba@v1
      with:
@ -66,6 +63,11 @@ jobs:
        cache-environment: true
        cache-downloads: true

+    - name: Install inhouse packages
+      run: |
+        pip install git+https://github.com/PyPSA/atlite.git@master git+https://github.com/PyPSA/powerplantmatching.git@master git+https://github.com/PyPSA/linopy.git@master
+      if: ${{ matrix.inhouse }} == 'master'
+
    - name: Set cache dates
      run: |
        echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV
@ -79,14 +81,10 @@ jobs:
        key: data-cutouts-${{ env.WEEK }}-${{ env.DATA_CACHE_NUMBER }}

    - name: Test snakemake workflow
-      run: |
-        snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime
-        snakemake -call all --configfile config/test/config.overnight.yaml --rerun-triggers=mtime
-        snakemake -call all --configfile config/test/config.myopic.yaml --rerun-triggers=mtime
-        snakemake -call all --configfile config/test/config.perfect.yaml --rerun-triggers=mtime
+      run: ./test.sh

    - name: Upload artifacts
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4.3.0
      with:
        name: resources-results
        path: |
@ -94,3 +92,4 @@ jobs:
          results
        if-no-files-found: warn
        retention-days: 1
+      if: matrix.os == 'ubuntu' && matrix.inhouse == 'stable'
--- a/.gitignore
+++ b/.gitignore
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0

@ -20,10 +20,18 @@ gurobi.log
 /notebooks
 /data
 /cutouts
-
+/tmp
 doc/_build

+/scripts/old
+/scripts/create_scenarios.py
+/config/create_scenarios.py
+
+config/config.yaml
+config/scenarios.yaml
+
 config.yaml
+config/config.yaml

 dconf
 /data/links_p_nom.csv
@ -46,31 +54,22 @@ data/costs_*.csv

 dask-worker-space/
 publications.jrc.ec.europa.eu/
+d1gam3xoknrgr2.cloudfront.net/

 *.org

 *.nc

 *~
-/scripts/old

 *.pyc
-/cutouts
-/tmp
-/pypsa

 *.xlsx

-config.yaml
-
-doc/_build
-
 *.xls

 *.geojson

 *.ipynb

-data/costs_*
-
 merger-todos.md
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@ -17,7 +17,7 @@ repos:

  # Sort package imports alphabetically
 - repo: https://github.com/PyCQA/isort
-  rev: 5.12.0
+  rev: 5.13.2
  hooks:
  - id: isort
    args: ["--profile", "black", "--filter-files"]
@ -50,8 +50,8 @@ repos:
  - id: blackdoc

  # Formatting with "black" coding style
- repo: https://github.com/psf/black
-  rev: 23.11.0
+- repo: https://github.com/psf/black-pre-commit-mirror
+  rev: 24.2.0
  hooks:
      # Format Python files
  - id: black
@ -67,14 +67,14 @@ repos:

  # Do YAML formatting (before the linter checks it for misses)
 - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
-  rev: v2.11.0
+  rev: v2.12.0
  hooks:
  - id: pretty-format-yaml
    args: [--autofix, --indent, "2", --preserve-quotes]

  # Format Snakemake rule / workflow files
 - repo: https://github.com/snakemake/snakefmt
-  rev: v0.8.5
+  rev: v0.10.0
  hooks:
  - id: snakefmt

@ -87,6 +87,6 @@ repos:

  # Check for FSFE REUSE compliance (licensing)
 - repo: https://github.com/fsfe/reuse-tool
-  rev: v2.1.0
+  rev: v3.0.1
  hooks:
  - id: reuse
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0

--- a/.reuse/dep5
+++ b/.reuse/dep5
@ -4,33 +4,33 @@ Upstream-Contact: Tom Brown <t.brown@tu-berlin.de>
 Source: https://github.com/pypsa/pypsa-eur

 Files: doc/img/*
-Copyright: 2019-2023 The PyPSA-Eur Authors
+Copyright: 2019-2024 The PyPSA-Eur Authors
 License: CC-BY-4.0

 Files: doc/data.csv
-Copyright: 2019-2023 The PyPSA-Eur Authors
+Copyright: 2019-2024 The PyPSA-Eur Authors
 License: CC-BY-4.0

 Files: doc/configtables/*
-Copyright: 2019-2023 The PyPSA-Eur Authors
+Copyright: 2019-2024 The PyPSA-Eur Authors
 License: CC-BY-4.0

 Files: data/*
-Copyright: 2017-2023 The PyPSA-Eur Authors
+Copyright: 2017-2024 The PyPSA-Eur Authors
 License: CC-BY-4.0

 Files: .github/*
-Copyright: 2019-2023 The PyPSA-Eur Authors
+Copyright: 2019-2024 The PyPSA-Eur Authors
 License: CC0-1.0

 Files: matplotlibrc
-Copyright: 2017-2023 The PyPSA-Eur Authors
+Copyright: 2017-2024 The PyPSA-Eur Authors
 License: CC0-1.0

 Files: borg-it
-Copyright: 2017-2023 The PyPSA-Eur Authors
+Copyright: 2017-2024 The PyPSA-Eur Authors
 License: CC0-1.0

 Files: graphics/*
-Copyright: 2017-2023 The PyPSA-Eur Authors
+Copyright: 2017-2024 The PyPSA-Eur Authors
 License: CC-BY-4.0
--- a/.sync-send
+++ b/.sync-send
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2021-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0

--- a/CITATION.cff
+++ b/CITATION.cff
@ -6,7 +6,7 @@ cff-version: 1.1.0
 message: "If you use this package, please cite it in the following way."
 title: "PyPSA-Eur: An open sector-coupled optimisation model of the European energy system"
 repository: https://github.com/pypsa/pypsa-eur
-version: 0.8.1
+version: 0.10.0
 license: MIT
 authors:
  - family-names: Brown
--- a/LICENSES/MIT.txt
+++ b/LICENSES/MIT.txt
@ -1,6 +1,6 @@
 MIT License

-Copyright 2017-2023 The PyPSA-Eur Authors
+Copyright 2017-2024 The PyPSA-Eur Authors

 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in
--- a/README.md
+++ b/README.md
@ -1,5 +1,5 @@
 <!--
-SPDX-FileCopyrightText: 2017-2023 The PyPSA-Eur Authors
+SPDX-FileCopyrightText: 2017-2024 The PyPSA-Eur Authors
 SPDX-License-Identifier: CC-BY-4.0
 -->

@ -61,9 +61,9 @@ The dataset consists of:

 - A grid model based on a modified [GridKit](https://github.com/bdw/GridKit)
  extraction of the [ENTSO-E Transmission System
-  Map](https://www.entsoe.eu/data/map/). The grid model contains 6763 lines
+  Map](https://www.entsoe.eu/data/map/). The grid model contains 7072 lines
  (alternating current lines at and above 220kV voltage level and all high
-  voltage direct current lines) and 3642 substations.
+  voltage direct current lines) and 3803 substations.
 - The open power plant database
  [powerplantmatching](https://github.com/FRESNA/powerplantmatching).
 - Electrical demand time series from the
@ -103,6 +103,6 @@ We strongly welcome anyone interested in contributing to this project. If you ha
 # Licence

 The code in PyPSA-Eur is released as free software under the
-[MIT License](https://opensource.org/licenses/MIT), see `LICENSE.txt`.
+[MIT License](https://opensource.org/licenses/MIT), see [`doc/licenses.rst`](doc/licenses.rst).
 However, different licenses and terms of use may apply to the various
 input data.
--- a/69
+++ b/69
@ -1,36 +1,51 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT

 from os.path import normpath, exists
 from shutil import copyfile, move, rmtree
-
-from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider
-
-HTTP = HTTPRemoteProvider()
+from pathlib import Path
+import yaml

 from snakemake.utils import min_version

-min_version("7.7")
-
-
-if not exists("config/config.yaml"):
-    copyfile("config/config.default.yaml", "config/config.yaml")
+min_version("8.5")
+
+from scripts._helpers import path_provider
+
+default_files = {
+    "config/config.default.yaml": "config/config.yaml",
+    "config/scenarios.template.yaml": "config/scenarios.yaml",
+}
+for template, target in default_files.items():
+    target = os.path.join(workflow.current_basedir, target)
+    template = os.path.join(workflow.current_basedir, template)
+    if not exists(target) and exists(template):
+        copyfile(template, target)


+configfile: "config/config.default.yaml"
 configfile: "config/config.yaml"


-COSTS = f"data/costs_{config['costs']['year']}.csv"
-ATLITE_NPROCESSES = config["atlite"].get("nprocesses", 4)
+run = config["run"]
+scenarios = run.get("scenarios", {})
+if run["name"] and scenarios.get("enable"):
+    fn = Path(scenarios["file"])
+    scenarios = yaml.safe_load(fn.read_text())
+    RDIR = "{run}/"
+    if run["name"] == "all":
+        config["run"]["name"] = list(scenarios.keys())
+elif run["name"]:
+    RDIR = run["name"] + "/"
+else:
+    RDIR = ""

-run = config.get("run", {})
-RDIR = run["name"] + "/" if run.get("name") else ""
-CDIR = RDIR if not run.get("shared_cutouts") else ""
+logs = path_provider("logs/", RDIR, run["shared_resources"])
+benchmarks = path_provider("benchmarks/", RDIR, run["shared_resources"])
+resources = path_provider("resources/", RDIR, run["shared_resources"])

-LOGS = "logs/" + RDIR
-BENCHMARKS = "benchmarks/" + RDIR
-RESOURCES = "resources/" + RDIR if not run.get("shared_resources") else "resources/"
+CDIR = "" if run["shared_cutouts"] else RDIR
 RESULTS = "results/" + RDIR


@ -73,10 +88,19 @@ if config["foresight"] == "perfect":

 rule all:
    input:
-        RESULTS + "graphs/costs.pdf",
+        expand(RESULTS + "graphs/costs.pdf", run=config["run"]["name"]),
    default_target: True


+rule create_scenarios:
+    output:
+        config["run"]["scenarios"]["file"],
+    conda:
+        "envs/retrieve.yaml"
+    script:
+        "config/create_scenarios.py"
+
+
 rule purge:
    run:
        import builtins
@ -97,9 +121,9 @@ rule dag:
    message:
        "Creating DAG of workflow."
    output:
-        dot=RESOURCES + "dag.dot",
-        pdf=RESOURCES + "dag.pdf",
-        png=RESOURCES + "dag.png",
+        dot=resources("dag.dot"),
+        pdf=resources("dag.pdf"),
+        png=resources("dag.png"),
    conda:
        "envs/environment.yaml"
    shell:
@ -125,6 +149,7 @@ rule sync:
    shell:
        """
        rsync -uvarh --ignore-missing-args --files-from=.sync-send . {params.cluster}
+        rsync -uvarh --no-g {params.cluster}/resources . || echo "No resources directory, skipping rsync"
        rsync -uvarh --no-g {params.cluster}/results . || echo "No results directory, skipping rsync"
        rsync -uvarh --no-g {params.cluster}/logs . || echo "No logs directory, skipping rsync"
        """
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@ -1,9 +1,9 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0

 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#top-level-configuration
-version: 0.8.1
+version: 0.10.0
 tutorial: false

 logging:
@ -21,6 +21,9 @@ remote:
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#run
 run:
  name: ""
+  scenarios:
+    enable: false
+    file: config/scenarios.yaml
  disable_progressbar: false
  shared_resources: false
  shared_cutouts: true
@ -44,7 +47,7 @@ scenario:
  opts:
  - ''
  sector_opts:
-  - Co2L0-3H-T-H-B-I-A-solar+p3-dist1
+  - Co2L0-3H-T-H-B-I-A-dist1
  planning_horizons:
  # - 2020
  # - 2030
@ -74,6 +77,7 @@ enable:
  retrieve_natura_raster: true
  custom_busmap: false

+
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#co2-budget
 co2_budget:
  2020: 0.701
@ -86,8 +90,10 @@ co2_budget:

 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#electricity
 electricity:
-  voltages: [220., 300., 380.]
+  voltages: [220., 300., 380., 500., 750.]
+  gaslimit_enable: false
  gaslimit: false
+  co2limit_enable: false
  co2limit: 7.75e+7
  co2base: 1.487e+9
  agg_p_nom_limits: data/agg_p_nom_minmax.csv
@ -108,8 +114,9 @@ electricity:
    Store: [battery, H2]
    Link: [] # H2 pipeline

-  powerplants_filter: (DateOut >= 2022 or DateOut != DateOut)
+  powerplants_filter: (DateOut >= 2023 or DateOut != DateOut) and not (Country == 'Germany' and Fueltype == 'Nuclear')
  custom_powerplants: false
+  everywhere_powerplants: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass]

  conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass]
  renewable_carriers: [solar, onwind, offwind-ac, offwind-dc, hydro]
@ -124,6 +131,10 @@ electricity:
      Onshore: [onwind]
      PV: [solar]

+  autarky:
+    enable: false
+    by_country: false
+
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#atlite
 atlite:
  default_cutout: europe-2013-era5
@ -135,14 +146,14 @@ atlite:
      # module: era5
    europe-2013-era5:
      module: era5 # in priority order
-      x: [-12., 35.]
+      x: [-12., 42.]
      y: [33., 72]
      dx: 0.3
      dy: 0.3
      time: ['2013', '2013']
    europe-2013-sarah:
      module: [sarah, era5] # in priority order
-      x: [-12., 45.]
+      x: [-12., 42.]
      y: [33., 65]
      dx: 0.2
      dy: 0.2
@ -158,45 +169,51 @@ renewable:
    resource:
      method: wind
      turbine: Vestas_V112_3MW
+      add_cutout_windspeed: true
    capacity_per_sqkm: 3
    # correction_factor: 0.93
    corine:
      grid_codes: [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32]
      distance: 1000
      distance_grid_codes: [1, 2, 3, 4, 5, 6]
+    luisa: false
+      # grid_codes: [1111, 1121, 1122, 1123, 1130, 1210, 1221, 1222, 1230, 1241, 1242]
+      # distance: 1000
+      # distance_grid_codes: [1111, 1121, 1122, 1123, 1130, 1210, 1221, 1222, 1230, 1241, 1242]
    natura: true
    excluder_resolution: 100
-    potential: simple # or conservative
    clip_p_max_pu: 1.e-2
  offwind-ac:
    cutout: europe-2013-era5
    resource:
      method: wind
-      turbine: NREL_ReferenceTurbine_5MW_offshore
+      turbine: NREL_ReferenceTurbine_2020ATB_5.5MW
+      add_cutout_windspeed: true
    capacity_per_sqkm: 2
    correction_factor: 0.8855
    corine: [44, 255]
+    luisa: false # [0, 5230]
    natura: true
    ship_threshold: 400
    max_depth: 50
    max_shore_distance: 30000
    excluder_resolution: 200
-    potential: simple # or conservative
    clip_p_max_pu: 1.e-2
  offwind-dc:
    cutout: europe-2013-era5
    resource:
      method: wind
-      turbine: NREL_ReferenceTurbine_5MW_offshore
+      turbine: NREL_ReferenceTurbine_2020ATB_5.5MW
+      add_cutout_windspeed: true
    capacity_per_sqkm: 2
    correction_factor: 0.8855
    corine: [44, 255]
+    luisa: false # [0, 5230]
    natura: true
    ship_threshold: 400
    max_depth: 50
    min_shore_distance: 30000
    excluder_resolution: 200
-    potential: simple # or conservative
    clip_p_max_pu: 1.e-2
  solar:
    cutout: europe-2013-sarah
@ -206,12 +223,12 @@ renewable:
      orientation:
        slope: 35.
        azimuth: 180.
-    capacity_per_sqkm: 1.7
+    capacity_per_sqkm: 5.1
    # correction_factor: 0.854337
    corine: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 26, 31, 32]
+    luisa: false # [1111, 1121, 1122, 1123, 1130, 1210, 1221, 1222, 1230, 1241, 1242, 1310, 1320, 1330, 1410, 1421, 1422, 2110, 2120, 2130, 2210, 2220, 2230, 2310, 2410, 2420, 3210, 3320, 3330]
    natura: true
    excluder_resolution: 100
-    potential: simple # or conservative
    clip_p_max_pu: 1.e-2
  hydro:
    cutout: europe-2013-era5
@ -235,10 +252,13 @@ lines:
    220.: "Al/St 240/40 2-bundle 220.0"
    300.: "Al/St 240/40 3-bundle 300.0"
    380.: "Al/St 240/40 4-bundle 380.0"
+    500.: "Al/St 240/40 4-bundle 380.0"
+    750.: "Al/St 560/50 4-bundle 750.0"
  s_max_pu: 0.7
  s_nom_max: .inf
-  max_extension: .inf
+  max_extension: 20000 #MW
  length_factor: 1.25
+  reconnect_crimea: true
  under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity
  dynamic_line_rating:
    activate: false
@ -251,7 +271,7 @@ lines:
 links:
  p_max_pu: 1.0
  p_nom_max: .inf
-  max_extension: .inf
+  max_extension: 30000 #MW
  include_tyndp: true
  under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity

@ -261,7 +281,7 @@ transformers:
  s_nom: 2000.
  type: ''

-# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#load
+# docs-load in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#load
 load:
  power_statistics: true
  interpolate_limit: 3
@ -286,6 +306,7 @@ pypsa_eur:
  - offwind-dc
  - solar
  - ror
+  - nuclear
  StorageUnit:
  - PHS
  - hydro
@ -293,9 +314,8 @@ pypsa_eur:

 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#energy
 energy:
-  energy_totals_year: 2011
+  energy_totals_year: 2019
  base_emissions_year: 1990
-  eurostat_report_year: 2016
  emissions: CO2

 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#biomass
@ -333,9 +353,10 @@ solar_thermal:

 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#existing-capacities
 existing_capacities:
-  grouping_years_power: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030]
+  grouping_years_power: [1960, 1965, 1970, 1975, 1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030]
  grouping_years_heat: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019] # these should not extend 2020
  threshold_capacity: 10
+  default_heating_lifetime: 20
  conventional_carriers:
  - lignite
  - coal
@ -344,15 +365,23 @@ existing_capacities:

 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#sector
 sector:
+  transport: true
+  heating: true
+  biomass: true
+  industry: true
+  agriculture: true
  district_heating:
    potential: 0.6
    progress:
      2020: 0.0
+      2025: 0.15
      2030: 0.3
+      2035: 0.45
      2040: 0.6
+      2045: 0.8
      2050: 1.0
    district_heating_loss: 0.15
-  cluster_heat_buses: false
+  cluster_heat_buses: true
  bev_dsm_restriction_value: 0.75
  bev_dsm_restriction_time: 7
  transport_heating_deadband_upper: 20.
@ -372,18 +401,27 @@ sector:
  v2g: true
  land_transport_fuel_cell_share:
    2020: 0
-    2030: 0.05
-    2040: 0.1
-    2050: 0.15
+    2025: 0
+    2030: 0
+    2035: 0
+    2040: 0
+    2045: 0
+    2050: 0
  land_transport_electric_share:
    2020: 0
-    2030: 0.25
-    2040: 0.6
-    2050: 0.85
+    2025: 0.15
+    2030: 0.3
+    2035: 0.45
+    2040: 0.7
+    2045: 0.85
+    2050: 1
  land_transport_ice_share:
    2020: 1
+    2025: 0.85
    2030: 0.7
+    2035: 0.55
    2040: 0.3
+    2045: 0.15
    2050: 0
  transport_fuel_cell_efficiency: 0.5
  transport_internal_combustion_efficiency: 0.3
@ -397,18 +435,27 @@ sector:
  shipping_hydrogen_liquefaction: false
  shipping_hydrogen_share:
    2020: 0
+    2025: 0
    2030: 0
+    2035: 0
    2040: 0
+    2045: 0
    2050: 0
  shipping_methanol_share:
    2020: 0
+    2025: 0.15
    2030: 0.3
+    2035: 0.5
    2040: 0.7
+    2045: 0.85
    2050: 1
  shipping_oil_share:
    2020: 1
+    2025: 0.85
    2030: 0.7
+    2035: 0.5
    2040: 0.3
+    2045: 0.15
    2050: 0
  shipping_methanol_efficiency: 0.46
  shipping_oil_efficiency: 0.40
@ -437,23 +484,28 @@ sector:
    decentral: 3
    central: 180
  boilers: true
+  resistive_heaters: true
  oil_boilers: false
  biomass_boiler: true
+  overdimension_individual_heating: 1.1  #to cover demand peaks bigger than data
  chp: true
  micro_chp: false
  solar_thermal: true
  solar_cf_correction: 0.788457  # =  >>> 1/1.2683
  marginal_cost_storage: 0. #1e-4
  methanation: true
-  helmeth: false
  coal_cc: false
  dac: true
  co2_vent: false
+  central_heat_vent: false
  allam_cycle: false
  hydrogen_fuel_cell: true
  hydrogen_turbine: false
  SMR: true
  SMR_cc: true
+  regional_methanol_demand: false
+  regional_oil_demand: false
+  regional_coal_demand: false
  regional_co2_sequestration_potential:
    enable: false
    attribute: 'conservative estimate Mt'
@ -466,6 +518,7 @@ sector:
  co2_sequestration_lifetime: 50
  co2_spatial: false
  co2network: false
+  co2_network_cost_factor: 1
  cc_fraction: 0.9
  hydrogen_underground_storage: true
  hydrogen_underground_storage_locations:
@ -473,14 +526,29 @@ sector:
  - nearshore    # within 50 km of sea
    # - offshore
  ammonia: false
-  min_part_load_fischer_tropsch: 0.9
-  min_part_load_methanolisation: 0.5
+  min_part_load_fischer_tropsch: 0.7
+  min_part_load_methanolisation: 0.3
+  min_part_load_methanation: 0.3
  use_fischer_tropsch_waste_heat: true
+  use_haber_bosch_waste_heat: true
+  use_methanolisation_waste_heat: true
+  use_methanation_waste_heat: true
  use_fuel_cell_waste_heat: true
-  use_electrolysis_waste_heat: false
+  use_electrolysis_waste_heat: true
+  electricity_transmission_grid: true
  electricity_distribution_grid: true
  electricity_distribution_grid_cost_factor: 1.0
  electricity_grid_connection: true
+  transmission_efficiency:
+    DC:
+      efficiency_static: 0.98
+      efficiency_per_1000km: 0.977
+    H2 pipeline:
+      efficiency_per_1000km: 1 # 0.982
+      compression_per_1000km: 0.018
+    gas pipeline:
+      efficiency_per_1000km: 1 #0.977
+      compression_per_1000km: 0.01
  H2_network: true
  gas_network: false
  H2_retrofit: false
@ -490,6 +558,7 @@ sector:
  gas_distribution_grid_cost_factor: 1.0
  biomass_spatial: false
  biomass_transport: false
+  biogas_upgrading_cc: false
  conventional_generation:
    OCGT: gas
  biomass_to_liquid: false
@ -547,14 +616,48 @@ industry:
  MWh_NH3_per_tNH3: 5.166
  MWh_CH4_per_tNH3_SMR: 10.8
  MWh_elec_per_tNH3_SMR: 0.7
-  MWh_H2_per_tNH3_electrolysis: 6.5
-  MWh_elec_per_tNH3_electrolysis: 1.17
+  MWh_H2_per_tNH3_electrolysis: 5.93
+  MWh_elec_per_tNH3_electrolysis: 0.2473
  MWh_NH3_per_MWh_H2_cracker: 1.46 # https://github.com/euronion/trace/blob/44a5ff8401762edbef80eff9cfe5a47c8d3c8be4/data/efficiencies.csv
  NH3_process_emissions: 24.5
  petrochemical_process_emissions: 25.5
-  HVC_primary_fraction: 1.
-  HVC_mechanical_recycling_fraction: 0.
-  HVC_chemical_recycling_fraction: 0.
+  #HVC primary/recycling based on values used in Neumann et al https://doi.org/10.1016/j.joule.2023.06.016, linearly interpolated between 2020 and 2050
+  #2020 recycling rates based on Agora https://static.agora-energiewende.de/fileadmin/Projekte/2021/2021_02_EU_CEAP/A-EW_254_Mobilising-circular-economy_study_WEB.pdf
+  #fractions refer to the total primary HVC production in 2020
+  #assumes 6.7 Mtplastics produced from recycling in 2020
+  HVC_primary_fraction:
+    2020: 1.0
+    2025: 0.9
+    2030: 0.8
+    2035: 0.7
+    2040: 0.6
+    2045: 0.5
+    2050: 0.4
+  HVC_mechanical_recycling_fraction:
+    2020: 0.12
+    2025: 0.15
+    2030: 0.18
+    2035: 0.21
+    2040: 0.24
+    2045: 0.27
+    2050: 0.30
+  HVC_chemical_recycling_fraction:
+    2020: 0.0
+    2025: 0.0
+    2030: 0.04
+    2035: 0.08
+    2040: 0.12
+    2045: 0.16
+    2050: 0.20
+  sector_ratios_fraction_future:
+    2020: 0.0
+    2025: 0.1
+    2030: 0.3
+    2035: 0.5
+    2040: 0.7
+    2045: 0.9
+    2050: 1.0
+  basic_chemicals_without_NH3_production_today: 69. #Mt/a, = 86 Mtethylene-equiv - 17 MtNH3
  HVC_production_today: 52.
  MWh_elec_per_tHVC_mechanical_recycling: 0.547
  MWh_elec_per_tHVC_chemical_recycling: 6.9
@ -571,7 +674,7 @@ industry:
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#costs
 costs:
  year: 2030
-  version: v0.6.0
+  version: v0.8.1
  rooftop_share: 0.14  # based on the potentials, assuming  (0.1 kW/m2 and 10 m2/person)
  social_discountrate: 0.02
  fill_values:
@ -597,10 +700,13 @@ costs:
    battery: 0.
    battery inverter: 0.
  emission_prices:
+    enable: false
    co2: 0.
+    co2_monthly_prices: false

 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#clustering
 clustering:
+  focus_weights: false
  simplify_network:
    to_substations: false
    algorithm: kmeans # choose from: [hac, kmeans]
@ -618,6 +724,14 @@ clustering:
      committable: any
      ramp_limit_up: max
      ramp_limit_down: max
+  temporal:
+    resolution_elec: false
+    resolution_sector: false
+
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#adjustments
+adjustments:
+  electricity: false
+  sector: false

 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#solving
 solving:
@ -629,14 +743,22 @@ solving:
    skip_iterations: true
    rolling_horizon: false
    seed: 123
+    custom_extra_functionality: "../data/custom_extra_functionality.py"
+    # io_api: "direct"  # Increases performance but only supported for the highs and gurobi solvers
    # options that go into the optimize function
    track_iterations: false
    min_iterations: 4
    max_iterations: 6
-    transmission_losses: 0
+    transmission_losses: 2
    linearized_unit_commitment: true
    horizon: 365

+  constraints:
+    CCL: false
+    EQ: false
+    BAU: false
+    SAFE: false
+
  solver:
    name: gurobi
    options: gurobi-default
@ -691,11 +813,15 @@ solving:
      solutiontype: 2 # non basic solution, ie no crossover
      barrier.convergetol: 1.e-5
      feasopt.tolerance: 1.e-6
+    copt-default:
+      Threads: 8
+      LpMethod: 2
+      Crossover: 0
    cbc-default: {} # Used in CI
    glpk-default: {} # Used in CI

-  mem: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2
-  walltime: "12:00:00"
+  mem_mb: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2
+  runtime: 6h #runtime in humanfriendly style https://humanfriendly.readthedocs.io/en/latest/

 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#plotting
 plotting:
@ -704,6 +830,13 @@ plotting:
    color_geomap:
      ocean: white
      land: white
+  projection:
+    name: "EqualEarth"
+    # See https://scitools.org.uk/cartopy/docs/latest/reference/projections.html for alternatives, for example:
+    # name: "LambertConformal"
+    # central_longitude: 10.
+    # central_latitude: 50.
+    # standard_parallels: [35, 65]
  eu_node_location:
    x: -5.5
    y: 46.
@ -726,6 +859,7 @@ plotting:
    H2: "Hydrogen Storage"
    lines: "Transmission Lines"
    ror: "Run of River"
+    load: "Load Shedding"
    ac: "AC"
    dc: "DC"

@ -749,7 +883,6 @@ plotting:
    hydroelectricity: '#298c81'
    PHS: '#51dbcc'
    hydro+PHS: "#08ad97"
-    wave: '#a7d4cf'
    # solar
    solar: "#f9d002"
    solar PV: "#f9d002"
@ -776,6 +909,7 @@ plotting:
    fossil gas: '#e05b09'
    natural gas: '#e05b09'
    biogas to gas: '#e36311'
+    biogas to gas CC: '#e51245'
    CCGT: '#a85522'
    CCGT marginal: '#a85522'
    allam: '#B98F76'
@ -877,12 +1011,14 @@ plotting:
    # heat demand
    Heat load: '#cc1f1f'
    heat: '#cc1f1f'
+    heat vent: '#aa3344'
    heat demand: '#cc1f1f'
    rural heat: '#ff5c5c'
    residential rural heat: '#ff7c7c'
    services rural heat: '#ff9c9c'
    central heat: '#cc1f1f'
    urban central heat: '#d15959'
+    urban central heat vent: '#a74747'
    decentral heat: '#750606'
    residential urban decentral heat: '#a33c3c'
    services urban decentral heat: '#cc1f1f'
@ -895,9 +1031,11 @@ plotting:
    air heat pump: '#36eb41'
    residential urban decentral air heat pump: '#48f74f'
    services urban decentral air heat pump: '#5af95d'
+    services rural air heat pump: '#5af95d'
    urban central air heat pump: '#6cfb6b'
    ground heat pump: '#2fb537'
    residential rural ground heat pump: '#48f74f'
+    residential rural air heat pump: '#48f74f'
    services rural ground heat pump: '#5af95d'
    Ambient: '#98eb9d'
    CHP: '#8a5751'
@ -945,7 +1083,6 @@ plotting:
    Sabatier: '#9850ad'
    methanation: '#c44ce6'
    methane: '#c44ce6'
-    helmeth: '#e899ff'
    # synfuels
    Fischer-Tropsch: '#25c49a'
    liquid: '#25c49a'
@ -960,6 +1097,7 @@ plotting:
    CO2 sequestration: '#f29dae'
    DAC: '#ff5270'
    co2 stored: '#f2385a'
+    co2 sequestered: '#f2682f'
    co2: '#f29dae'
    co2 vent: '#ffd4dc'
    CO2 pipeline: '#f5627f'
@ -993,3 +1131,4 @@ plotting:
    DC: "#8a1caf"
    DC-DC: "#8a1caf"
    DC link: "#8a1caf"
+    load: "#dd2e23"
--- a/config/config.entsoe-all.yaml
+++ b/config/config.entsoe-all.yaml
@ -0,0 +1,43 @@
+# SPDX-FileCopyrightText: 2017-2024 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: CC0-1.0
+
+run:
+  name: "entsoe-all"
+  disable_progressbar: true
+  shared_resources: false
+  shared_cutouts: true
+
+scenario:
+  simpl:
+  - ''
+  ll:
+  - vopt
+  clusters:
+  - 39
+  - 128
+  - 256
+  opts:
+  - ''
+  sector_opts:
+  - ''
+  planning_horizons:
+  - ''
+
+# TODO add Turkey (TR)
+countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MD', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK', 'UA']
+
+electricity:
+  custom_powerplants: true
+  co2limit: 9.59e+7
+  co2base: 1.918e+9
+
+lines:
+  reconnect_crimea: true
+
+enable:
+  retrieve: true
+  retrieve_databundle: true
+  retrieve_sector_databundle: false
+  retrieve_cost_data: true
+  retrieve_cutout: true
--- a/config/config.perfect.yaml
+++ b/config/config.perfect.yaml
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 run:
@ -19,13 +19,16 @@ scenario:
  opts:
  - ''
  sector_opts:
-  - 1p5-4380H-T-H-B-I-A-solar+p3-dist1
-  - 1p7-4380H-T-H-B-I-A-solar+p3-dist1
-  - 2p0-4380H-T-H-B-I-A-solar+p3-dist1
+  - 1p5-4380H-T-H-B-I-A-dist1
+  - 1p7-4380H-T-H-B-I-A-dist1
+  - 2p0-4380H-T-H-B-I-A-dist1
  planning_horizons:
  - 2020
+  - 2025
  - 2030
+  - 2035
  - 2040
+  - 2045
  - 2050


--- a/config/config.validation.yaml
+++ b/config/config.validation.yaml
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 run:
--- a/config/create_scenarios.py
+++ b/config/create_scenarios.py
@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2023-2024 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+# This script helps to generate a scenarios.yaml file for PyPSA-Eur.
+# You can modify the template to your needs and define all possible combinations of config values that should be considered.
+
+if "snakemake" in globals():
+    filename = snakemake.output[0]
+else:
+    filename = "../config/scenarios.yaml"
+
+import itertools
+
+# Insert your config values that should be altered in the template.
+# Change `config_section` and `config_section2` to the actual config sections.
+template = """
+scenario{scenario_number}:
+    config_section:
+        config_key: {config_value}
+    config_section2:
+        config_key2: {config_value2}
+"""
+
+# Define all possible combinations of config values.
+# This must define all config values that are used in the template.
+config_values = dict(config_value=["true", "false"], config_value2=[1, 2, 3, 4])
+
+combinations = [
+    dict(zip(config_values.keys(), values))
+    for values in itertools.product(*config_values.values())
+]
+
+with open(filename, "w") as f:
+    for i, config in enumerate(combinations):
+        f.write(template.format(scenario_number=i, **config))
--- a/config/scenarios.template.yaml
+++ b/config/scenarios.template.yaml
@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+# This file is used to define the scenarios that are run by snakemake. Each entry on the first level is a scenario. Each scenario can contain configuration overrides with respect to the config/config.yaml settings.
+#
+# Example
+#
+# custom-scenario: # name of the scenario
+#   electricity:
+#       renewable_carriers: [wind, solar] # override the list of renewable carriers
+
+normal:
+  electricity:
+    renewable_carriers:
+    - solar
+    - onwind
+    - offwind-ac
+    - offwind-dc
+    - hydro
+
+no-offwind:
+  electricity:
+    renewable_carriers:
+    - solar
+    - onwind
+    - hydro
--- a/config/test/config.electricity.yaml
+++ b/config/test/config.electricity.yaml
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0

@ -8,14 +8,14 @@ tutorial: true
 run:
  name: "test-elec" # use this to keep track of runs with different settings
  disable_progressbar: true
-  shared_resources: true
+  shared_resources: "test"
  shared_cutouts: true

 scenario:
  clusters:
  - 5
  opts:
-  - Co2L-24H
+  - Co2L-24h

 countries: ['BE']

--- a/config/test/config.myopic.yaml
+++ b/config/test/config.myopic.yaml
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0

@ -7,7 +7,7 @@ tutorial: true
 run:
  name: "test-sector-myopic"
  disable_progressbar: true
-  shared_resources: true
+  shared_resources: "test"
  shared_cutouts: true

 foresight: myopic
@ -18,7 +18,7 @@ scenario:
  clusters:
  - 5
  sector_opts:
-  - 24H-T-H-B-I-A-solar+p3-dist1
+  - 24h-T-H-B-I-A-dist1
  planning_horizons:
  - 2030
  - 2040
@ -30,6 +30,9 @@ snapshots:
  start: "2013-03-01"
  end: "2013-03-08"

+sector:
+  central_heat_vent: true
+
 electricity:
  co2limit: 100.e+6

--- a/config/test/config.overnight.yaml
+++ b/config/test/config.overnight.yaml
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0

@ -7,7 +7,7 @@ tutorial: true
 run:
  name: "test-sector-overnight"
  disable_progressbar: true
-  shared_resources: true
+  shared_resources: "test"
  shared_cutouts: true


@ -17,7 +17,7 @@ scenario:
  clusters:
  - 5
  sector_opts:
-  - CO2L0-24H-T-H-B-I-A-solar+p3-dist1
+  - CO2L0-24h-T-H-B-I-A-dist1
  planning_horizons:
  - 2030

--- a/config/test/config.perfect.yaml
+++ b/config/test/config.perfect.yaml
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0

@ -7,7 +7,7 @@ tutorial: true
 run:
  name: "test-sector-perfect"
  disable_progressbar: true
-  shared_resources: true
+  shared_resources: "test"
  shared_cutouts: true

 foresight: perfect
@ -18,7 +18,7 @@ scenario:
  clusters:
  - 5
  sector_opts:
-  - 8760H-T-H-B-I-A-solar+p3-dist1
+  - 8760h-T-H-B-I-A-dist1
  planning_horizons:
  - 2030
  - 2040
@ -44,6 +44,7 @@ electricity:
 sector:
  min_part_load_fischer_tropsch: 0
  min_part_load_methanolisation: 0
+
 atlite:
  default_cutout: be-03-2013-era5
  cutouts:
--- a/config/test/config.scenarios.yaml
+++ b/config/test/config.scenarios.yaml
@ -0,0 +1,60 @@
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: CC0-1.0
+
+tutorial: true
+
+run:
+  name:
+  - test-elec-no-offshore-wind
+  - test-elec-no-onshore-wind
+  scenarios:
+    enable: true
+    file: "config/test/scenarios.yaml"
+  disable_progressbar: true
+  shared_resources: base
+  shared_cutouts: true
+
+scenario:
+  clusters:
+  - 5
+  opts:
+  - Co2L-24H
+
+countries: ['BE']
+
+snapshots:
+  start: "2013-03-01"
+  end: "2013-03-08"
+
+electricity:
+  extendable_carriers:
+    Generator: [OCGT]
+    StorageUnit: [battery, H2]
+    Store: []
+
+atlite:
+  default_cutout: be-03-2013-era5
+  cutouts:
+    be-03-2013-era5:
+      module: era5
+      x: [4., 15.]
+      y: [46., 56.]
+      time: ["2013-03-01", "2013-03-08"]
+
+renewable:
+  onwind:
+    cutout: be-03-2013-era5
+  offwind-ac:
+    cutout: be-03-2013-era5
+    max_depth: false
+  offwind-dc:
+    cutout: be-03-2013-era5
+    max_depth: false
+  solar:
+    cutout: be-03-2013-era5
+
+solving:
+  solver:
+    name: glpk
+    options: "glpk-default"
--- a/config/test/scenarios.yaml
+++ b/config/test/scenarios.yaml
@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: CC0-1.0
+
+test-elec-no-offshore-wind:
+  electricity:
+    renewable_carriers: [solar, onwind]
+
+test-elec-no-onshore-wind:
+  electricity:
+    renewable_carriers: [solar, offwind-ac, offwind-dc]
--- a/data/GDP_PPP_30arcsec_v3_mapped_default.csv
+++ b/data/GDP_PPP_30arcsec_v3_mapped_default.csv
@ -0,0 +1,151 @@
+name,GDP_PPP,country
+3140,632728.0438507323,MD
+3139,806541.9318093687,MD
+3142,1392454.6690911907,MD
+3152,897871.2903553953,MD
+3246,645554.8588933202,MD
+7049,1150156.4449477682,MD
+1924,162285.16792916053,UA
+1970,751970.6071848695,UA
+2974,368873.75840156944,UA
+2977,294847.85539198935,UA
+2979,197988.13680768458,UA
+2980,301371.2491126519,UA
+3031,56925.21878805953,UA
+3032,139395.18279351242,UA
+3033,145377.8061037629,UA
+3035,52282.83655208812,UA
+3036,497950.25890516065,UA
+3037,1183293.1987702171,UA
+3038,255005.98207636533,UA
+3039,224711.50098325178,UA
+3040,342959.943226467,UA
+3044,69119.31486955672,UA
+3045,246273.65986119965,UA
+3047,146742.08407299497,UA
+3049,107265.7028733467,UA
+3050,1126147.985259493,UA
+3051,69833.56303043803,UA
+3052,67230.88206577855,UA
+3053,27019.224685201345,UA
+3054,260571.47337292184,UA
+3055,88760.94152915622,UA
+3056,101368.26196568517,UA
+3058,55752.92329667119,UA
+3059,89024.37880630122,UA
+3062,358411.291265149,UA
+3064,75081.64142862396,UA
+3065,158101.42949135564,UA
+3066,83763.89576442329,UA
+3068,173474.51218344545,UA
+3069,60327.01572375589,UA
+3070,18073.687271955278,UA
+3071,249069.43314695224,UA
+3072,220707.35700825177,UA
+3073,61342.30137462664,UA
+3074,254235.98867635374,UA
+3077,769558.9832370486,UA
+3078,132674.2315809836,UA
+3079,1388517.1478032232,UA
+3080,1861003.8718246964,UA
+3082,140123.73854745473,UA
+3083,834887.5595419679,UA
+3084,1910795.5590558557,UA
+3086,93828.36549170096,UA
+3088,347197.65113392205,UA
+3089,3754718.141734592,UA
+3090,521912.69768585655,UA
+3093,232818.05269714879,UA
+3095,435376.20361377904,UA
+3099,345596.5288937008,UA
+3100,175689.10947424968,UA
+3105,538438.9311459162,UA
+3107,88096.86032871014,UA
+3108,79847.68447063807,UA
+3109,348504.73449373,UA
+3144,71657.0165675802,UA
+3146,80342.05037424155,UA
+3158,74465.12922576343,UA
+3164,3102112.2672631275,UA
+3165,65215.04081671433,UA
+3166,413924.2225725632,UA
+3167,135060.0056434935,UA
+3168,54980.442979330146,UA
+3170,29584.879122227037,UA
+3171,142780.68163047134,UA
+3172,40436.63814695243,UA
+3173,1253342.1790126422,UA
+3174,173842.03139155387,UA
+3176,65699.76352408895,UA
+3177,143591.75419817626,UA
+3178,56434.04525832523,UA
+3179,389996.1670051216,UA
+3180,138452.84503524794,UA
+3181,67402.59500436619,UA
+3184,51204.293695376415,UA
+3185,46867.82356528432,UA
+3186,103892.35612417295,UA
+3187,193668.91476930346,UA
+3189,54584.176457692694,UA
+3190,219077.64942830536,UA
+3197,88516.52699983507,UA
+3198,298166.8272673622,UA
+3199,61334.952541812374,UA
+3229,175692.61136747137,UA
+3230,106722.62773321665,UA
+3236,61542.06264321315,UA
+3241,83752.90489164277,UA
+4301,48419.52825967164,UA
+4305,147759.74280349456,UA
+4306,53156.905740992224,UA
+4315,218025.78516351627,UA
+4317,155240.40554731718,UA
+4318,1342144.2459407183,UA
+4319,91669.1449633853,UA
+4321,85852.49282415409,UA
+4347,67938.7698430624,UA
+4357,20064.979012172935,UA
+4360,47840.51245168512,UA
+4361,55580.924388032574,UA
+4362,165753.82588729708,UA
+4363,46390.2448142152,UA
+4365,96265.47592938849,UA
+4366,272003.25510057947,UA
+4367,80878.50229245829,UA
+4370,330072.35444044066,UA
+4371,7707066.181975477,UA
+4373,2019766.7891575783,UA
+4374,985354.331818515,UA
+4377,230805.08833664874,UA
+4382,125670.67125287943,UA
+4383,46914.065511740075,UA
+4384,48020.804310510954,UA
+4385,55612.34707641123,UA
+4387,74558.3475791577,UA
+4388,245243.33449409154,UA
+4389,95696.56767732685,UA
+4391,251085.7523045193,UA
+4401,66375.82996856027,UA
+4403,111954.41038437477,UA
+4405,46911.68560148837,UA
+4408,150782.51691456966,UA
+4409,112776.7399582134,UA
+4410,153076.56860965435,UA
+4412,192629.31238456024,UA
+4413,181295.3120834606,UA
+4414,995694.9413199169,UA
+4416,157640.7868989174,UA
+4418,77580.20674809469,UA
+4420,122320.99275223716,UA
+4424,184891.10924920067,UA
+4425,84486.75974340564,UA
+4431,50485.84380961137,UA
+4435,231040.45446464577,UA
+4436,81222.18707585508,UA
+4438,114819.76472988473,UA
+4439,76839.1052178896,UA
+4440,135337.0313562152,UA
+4441,49159.485269198034,UA
+7031,42001.73757065917,UA
+7059,159790.48382874,UA
+7063,39599.10564971086,UA
--- a/data/custom_extra_functionality.py
+++ b/data/custom_extra_functionality.py
@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2023- The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+
+def custom_extra_functionality(n, snapshots, snakemake):
+    """
+    Add custom extra functionality constraints.
+    """
+    pass
--- a/data/custom_powerplants.csv
+++ b/data/custom_powerplants.csv
@ -1 +1,37 @@
-Name,Fueltype,Technology,Set,Country,Capacity,Efficiency,Duration,Volume_Mm3,DamHeight_m,YearCommissioned,Retrofit,lat,lon,projectID,YearDecommissioning
+,Name,Fueltype,Technology,Set,Country,Capacity,Efficiency,Duration,Volume_Mm3,DamHeight_m,StorageCapacity_MWh,DateIn,DateRetrofit,DateMothball,DateOut,lat,lon,EIC,projectID
+1266,Khmelnitskiy,Nuclear,,PP,UA,1901.8916595755832,,0.0,0.0,0.0,0.0,1988.0,2005.0,,,50.3023,26.6466,[nan],"{'GEO': ['GEO3842'], 'GPD': ['WRI1005111'], 'CARMA': ['CARMA22000']}"
+1268,Kaniv,Hydro,Reservoir,PP,UA,452.1656050955414,,0.0,0.0,0.0,0.0,1972.0,2003.0,,,49.76653,31.47165,[nan],"{'GEO': ['GEO43017'], 'GPD': ['WRI1005122'], 'CARMA': ['CARMA21140']}"
+1269,Kahovska kakhovka,Hydro,Reservoir,PP,UA,352.45222929936307,,0.0,0.0,0.0,0.0,1955.0,1956.0,,,46.77858,33.36965,[nan],"{'GEO': ['GEO43018'], 'GPD': ['WRI1005118'], 'CARMA': ['CARMA20855']}"
+1347,Kharkiv,Natural Gas,Steam Turbine,CHP,UA,494.94274967602314,,0.0,0.0,0.0,0.0,1979.0,1980.0,,,49.9719,36.107,[nan],"{'GEO': ['GEO43027'], 'GPD': ['WRI1005126'], 'CARMA': ['CARMA21972']}"
+1348,Kremenchuk,Hydro,Reservoir,PP,UA,617.0382165605096,,0.0,0.0,0.0,0.0,1959.0,1960.0,,,49.07759,33.2505,[nan],"{'GEO': ['GEO43019'], 'GPD': ['WRI1005121'], 'CARMA': ['CARMA23072']}"
+1377,Krivorozhskaya,Hard Coal,Steam Turbine,PP,UA,2600.0164509342876,,0.0,0.0,0.0,0.0,1965.0,1992.0,,,47.5432,33.6583,[nan],"{'GEO': ['GEO42989'], 'GPD': ['WRI1005100'], 'CARMA': ['CARMA23176']}"
+1407,Zmiyevskaya zmiivskaya,Hard Coal,Steam Turbine,PP,UA,2028.3816283884514,,0.0,0.0,0.0,0.0,1960.0,2005.0,,,49.5852,36.5231,[nan],"{'GEO': ['GEO42999'], 'GPD': ['WRI1005103'], 'CARMA': ['CARMA51042']}"
+1408,Pridneprovskaya,Hard Coal,Steam Turbine,CHP,UA,1627.3152609570984,,0.0,0.0,0.0,0.0,1959.0,1966.0,,,48.4051,35.1131,[nan],"{'GEO': ['GEO42990'], 'GPD': ['WRI1005102'], 'CARMA': ['CARMA35874']}"
+1409,Kurakhovskaya,Hard Coal,Steam Turbine,PP,UA,1371.0015824607397,,0.0,0.0,0.0,0.0,1972.0,2003.0,,,47.9944,37.24022,[nan],"{'GEO': ['GEO42994'], 'GPD': ['WRI1005104'], 'CARMA': ['CARMA23339']}"
+1410,Dobrotvorsky,Hard Coal,Steam Turbine,PP,UA,553.1949895604868,,0.0,0.0,0.0,0.0,1960.0,1964.0,,,50.2133,24.375,[nan],"{'GEO': ['GEO42992'], 'GPD': ['WRI1005096'], 'CARMA': ['CARMA10971']}"
+1422,Zuyevskaya,Hard Coal,Steam Turbine,PP,UA,1147.87960333801,,0.0,0.0,0.0,0.0,1982.0,2007.0,,,48.0331,38.28615,[nan],"{'GEO': ['GEO42995'], 'GPD': ['WRI1005106'], 'CARMA': ['CARMA51083']}"
+1423,Zaporozhye,Nuclear,,PP,UA,5705.67497872675,,0.0,0.0,0.0,0.0,1985.0,1996.0,,,47.5119,34.5863,[nan],"{'GEO': ['GEO6207'], 'GPD': ['WRI1005114'], 'CARMA': ['CARMA50875']}"
+1424,Trypilska,Hard Coal,Steam Turbine,PP,UA,1659.5849686814602,,0.0,0.0,0.0,0.0,1969.0,1972.0,,,50.1344,30.7468,[nan],"{'GEO': ['GEO43000'], 'GPD': ['WRI1005099'], 'CARMA': ['CARMA46410']}"
+1425,Tashlyk,Hydro,Pumped Storage,Store,UA,285.55968954109585,,0.0,0.0,0.0,0.0,2006.0,2007.0,,,47.7968,31.1811,[nan],"{'GEO': ['GEO43025'], 'GPD': ['WRI1005117'], 'CARMA': ['CARMA44696']}"
+1426,Starobeshivska,Hard Coal,Steam Turbine,PP,UA,1636.5351774497733,,0.0,0.0,0.0,0.0,1961.0,1967.0,,,47.7997,38.00612,[nan],"{'GEO': ['GEO43003'], 'GPD': ['WRI1005105'], 'CARMA': ['CARMA43083']}"
+1427,South,Nuclear,,PP,UA,2852.837489363375,,0.0,0.0,0.0,0.0,1983.0,1989.0,,,47.812,31.22,[nan],"{'GEO': ['GEO5475'], 'GPD': ['WRI1005113'], 'CARMA': ['CARMA42555']}"
+1428,Rovno rivne,Nuclear,,PP,UA,2695.931427448389,,0.0,0.0,0.0,0.0,1981.0,2006.0,,,51.3245,25.89744,[nan],"{'GEO': ['GEO5174'], 'GPD': ['WRI1005112'], 'CARMA': ['CARMA38114']}"
+1429,Ladyzhinska,Hard Coal,Steam Turbine,PP,UA,1659.5849686814602,,0.0,0.0,0.0,0.0,1970.0,1971.0,,,48.706,29.2202,[nan],"{'GEO': ['GEO42993'], 'GPD': ['WRI1005098'], 'CARMA': ['CARMA24024']}"
+1430,Kiev,Hydro,Pumped Storage,PP,UA,635.8694635681177,,0.0,0.0,0.0,0.0,1964.0,1972.0,,,50.5998,30.501,"[nan, nan]","{'GEO': ['GEO43024', 'GEO43023'], 'GPD': ['WRI1005123', 'WRI1005124'], 'CARMA': ['CARMA23516', 'CARMA23517']}"
+2450,Cet chisinau,Natural Gas,,PP,MD,306.0,,0.0,0.0,0.0,0.0,,,,,47.027550000000005,28.8801,"[nan, nan]","{'GPD': ['WRI1002985', 'WRI1002984'], 'CARMA': ['CARMA8450', 'CARMA8451']}"
+2460,Hydropower che costesti,Hydro,,PP,MD,16.0,,0.0,0.0,0.0,0.0,1978.0,,,,47.8381,27.2246,[nan],"{'GPD': ['WRI1002987'], 'CARMA': ['CARMA9496']}"
+2465,Moldavskaya gres,Hard Coal,,PP,MD,2520.0,,0.0,0.0,0.0,0.0,,,,,46.6292,29.9407,[nan],"{'GPD': ['WRI1002989'], 'CARMA': ['CARMA28979']}"
+2466,Hydropower dubasari,Hydro,,PP,MD,48.0,,0.0,0.0,0.0,0.0,,,,,47.2778,29.123,[nan],"{'GPD': ['WRI1002988'], 'CARMA': ['CARMA11384']}"
+2676,Cet nord balti,Natural Gas,,PP,MD,24.0,,0.0,0.0,0.0,0.0,,,,,47.7492,27.8938,[nan],"{'GPD': ['WRI1002986'], 'CARMA': ['CARMA3071']}"
+2699,Dniprodzerzhynsk,Hydro,Reservoir,PP,UA,360.3503184713376,,0.0,0.0,0.0,0.0,1963.0,1964.0,,,48.5485,34.541015,[nan],"{'GEO': ['GEO43020'], 'GPD': ['WRI1005119']}"
+2707,Burshtynska tes,Hard Coal,Steam Turbine,PP,UA,2212.779958241947,,0.0,0.0,0.0,0.0,1965.0,1984.0,,,49.21038,24.66654,[nan],"{'GEO': ['GEO42991'], 'GPD': ['WRI1005097']}"
+2708,Danipro dnieper,Hydro,Reservoir,PP,UA,1484.8407643312103,,0.0,0.0,0.0,0.0,1932.0,1947.0,,,47.86944,35.08611,[nan],"{'GEO': ['GEO43016'], 'GPD': ['WRI1005120']}"
+2709,Dniester,Hydro,Pumped Storage,Store,UA,612.7241020616891,,0.0,0.0,0.0,0.0,2009.0,2011.0,,,48.51361,27.47333,[nan],"{'GEO': ['GEO43022'], 'GPD': ['WRI1005116', 'WRI1005115']}"
+2710,Kiev,Natural Gas,Steam Turbine,CHP,UA,458.2803237740955,,0.0,0.0,0.0,0.0,1982.0,1984.0,,,50.532,30.6625,[nan],"{'GEO': ['GEO42998'], 'GPD': ['WRI1005125']}"
+2712,Luganskaya,Hard Coal,Steam Turbine,PP,UA,1060.2903966575996,,0.0,0.0,0.0,0.0,1962.0,1969.0,,,48.74781,39.2624,[nan],"{'GEO': ['GEO42996'], 'GPD': ['WRI1005110']}"
+2713,Slavyanskaya,Hard Coal,Steam Turbine,PP,UA,737.5933194139823,,0.0,0.0,0.0,0.0,1971.0,1971.0,,,48.872,37.76567,[nan],"{'GEO': ['GEO43002'], 'GPD': ['WRI1005109']}"
+2714,Vuhlehirska uglegorskaya,Hard Coal,Steam Turbine,PP,UA,3319.1699373629203,,0.0,0.0,0.0,0.0,1972.0,1977.0,,,48.4633,38.20328,[nan],"{'GEO': ['GEO43001'], 'GPD': ['WRI1005107']}"
+2715,Zaporiska,Hard Coal,Steam Turbine,PP,UA,3319.1699373629203,,0.0,0.0,0.0,0.0,1972.0,1977.0,,,47.5089,34.6253,[nan],"{'GEO': ['GEO42988'], 'GPD': ['WRI1005101']}"
+3678,Mironovskaya,Hard Coal,,PP,UA,815.0,,0.0,0.0,0.0,0.0,,,,,48.3407,38.4049,[nan],"{'GPD': ['WRI1005108'], 'CARMA': ['CARMA28679']}"
+3679,Kramatorskaya,Hard Coal,,PP,UA,120.0,,0.0,0.0,0.0,0.0,1974.0,,,,48.7477,37.5723,[nan],"{'GPD': ['WRI1075856'], 'CARMA': ['CARMA54560']}"
+3680,Chernihiv,Hard Coal,,PP,UA,200.0,,0.0,0.0,0.0,0.0,1968.0,,,,51.455,31.2602,[nan],"{'GPD': ['WRI1075853'], 'CARMA': ['CARMA8190']}"
--- a/data/eia_hydro_annual_generation.csv
+++ b/data/eia_hydro_annual_generation.csv
@ -1,50 +1,53 @@
-https://www.eia.gov/international/data/world/electricity/electricity-generation?pd=2&p=000000000000000000000000000000g&u=1&f=A&v=mapbubble&a=-&i=none&vo=value&t=R&g=000000000000002&l=73-1028i008017kg6368g80a4k000e0ag00gg0004g8g0ho00g000400008&s=315532800000&e=1577836800000&ev=false&
-Report generated on: 03-28-2022 11:20:48
-"API","","1980","1981","1982","1983","1984","1985","1986","1987","1988","1989","1990","1991","1992","1993","1994","1995","1996","1997","1998","1999","2000","2001","2002","2003","2004","2005","2006","2007","2008","2009","2010","2011","2012","2013","2014","2015","2016","2017","2018","2019","2020"
-"","hydroelectricity net generation (billion kWh)","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","",""
-"INTL.33-12-EURO-BKWH.A","    Europe","458.018","464.155","459.881","473.685","481.241","476.739","459.535","491.085","534.517","465.365","474.466","475.47","509.041","526.448","531.815","543.743","529.114164","543.845616","562.441501","569.308453","591.206662","587.371195","541.542535","506.19703","544.536443","545.176179","537.335934","540.934407","567.557921","564.244482","619.96477","543.05273","600.46622","631.86431","619.59229","615.53013","629.98906","562.59258","619.31106","610.62616","670.925"
-"INTL.33-12-ALB-BKWH.A","        Albania","2.919","3.018","3.093","3.167","3.241","3.315","3.365","3.979","3.713","3.846","2.82","3.483","3.187","3.281","3.733","4.162","5.669","4.978","4.872","5.231","4.548","3.519","3.477","5.117","5.411","5.319","4.951","2.76","3.759","5.201","7.49133","4.09068","4.67775","6.88941","4.67676","5.83605","7.70418","4.47975","8.46648","5.15394","5.281"
-"INTL.33-12-AUT-BKWH.A","        Austria","28.501","30.008","29.893","29.577","28.384","30.288","30.496","25.401","35.151","34.641","31.179","31.112","34.483","36.336","35.349","36.696","33.874","35.744","36.792","40.292","41.418","40.05","39.825","32.883","36.394","36.31","35.48","36.732","37.969","40.487","36.466","32.511","41.862","40.138","39.001","35.255","37.954","36.462","35.73","40.43655","45.344"
-"INTL.33-12-BEL-BKWH.A","        Belgium","0.274","0.377","0.325","0.331","0.348","0.282","0.339","0.425","0.354","0.3","0.263","0.226","0.338","0.252","0.342","0.335","0.237","0.30195","0.38511","0.338","0.455","0.437","0.356","0.245","0.314","0.285","0.355","0.385","0.406","0.325","0.298","0.193","0.353","0.376","0.289","0.314","0.367","0.268","0.311","0.108","1.29"
-"INTL.33-12-BIH-BKWH.A","        Bosnia and Herzegovina","--","--","--","--","--","--","--","--","--","--","--","--","3.374","2.343","3.424","3.607","5.104","4.608","4.511","5.477","5.043","5.129","5.215","4.456","5.919","5.938","5.798","3.961","4.818","6.177","7.946","4.343","4.173","7.164","5.876","5.495","5.585","3.7521","6.35382","6.02019","6.1"
-"INTL.33-12-BGR-BKWH.A","        Bulgaria","3.674","3.58","3.018","3.318","3.226","2.214","2.302","2.512","2.569","2.662","1.859","2.417","2.042","1.923","1.453","2.291","2.89","2.726","3.066","2.725","2.646","1.72","2.172","2.999","3.136","4.294","4.196","2.845","2.796","3.435","4.98168","2.84328","3.14622","3.99564","4.55598","5.59845","3.8412","2.79972","5.09553","3.34917","3.37"
-"INTL.33-12-HRV-BKWH.A","        Croatia","--","--","--","--","--","--","--","--","--","--","--","--","4.298","4.302","4.881","5.212","7.156","5.234","5.403","6.524","5.794","6.482","5.311","4.827","6.888","6.27","5.94","4.194","5.164","6.663","9.035","4.983","4.789","8.536","8.917","6.327","6.784","5.255","7.62399","5.87268","3.4"
-"INTL.33-12-CYP-BKWH.A","        Cyprus","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0"
-"INTL.33-12-CZE-BKWH.A","        Czech Republic","--","--","--","--","--","--","--","--","--","--","--","--","--","1.355","1.445","1.982","1.949","1.68201","1.382","1.664","1.7404","2.033","2.467","1.369","1.999","2.356","2.525","2.068","2.004","2.405","2.775","1.95","2.107","2.704","1.909","1.779","1.983","1.852","1.615","1.98792","3.4"
-"INTL.33-12-DNK-BKWH.A","        Denmark","0.03","0.031","0.028","0.036","0.028","0.027","0.029","0.029","0.032","0.027","0.027","0.026","0.028","0.027","0.033","0.03","0.019","0.019","0.02673","0.031","0.03","0.028","0.032","0.021","0.027","0.023","0.023","0.028","0.026","0.019","0.021","0.017","0.017","0.013","0.015","0.018","0.019","0.018","0.015","0.01584","0.02"
-"INTL.33-12-EST-BKWH.A","        Estonia","--","--","--","--","--","--","--","--","--","--","--","--","0.001","0.001","0.003","0.002","0.002","0.003","0.004","0.004","0.005","0.007","0.006","0.013","0.022","0.022","0.014","0.021","0.028","0.032","0.027","0.03","0.042","0.026","0.027","0.027","0.035","0.026","0.015","0.01881","0.04"
-"INTL.33-12-FRO-BKWH.A","        Faroe Islands","0.049","0.049","0.049","0.049","0.049","0.049","0.049","0.049","0.062","0.071","0.074","0.074","0.083","0.073","0.075","0.075","0.069564","0.075066","0.076501","0.069453","0.075262","0.075195","0.095535","0.08483","0.093443","0.097986","0.099934","0.103407","0.094921","0.091482","0.06676","0.092","0.099","0.091","0.121","0.132","0.105","0.11","0.107","0.102","0.11"
-"INTL.33-12-FIN-BKWH.A","        Finland","10.115","13.518","12.958","13.445","13.115","12.211","12.266","13.658","13.229","12.9","10.75","13.065","14.956","13.341","11.669","12.796","11.742","12.11958","14.9","12.652","14.513","13.073","10.668","9.495","14.919","13.646","11.379","14.035","16.941","12.559","12.743","12.278","16.667","12.672","13.24","16.584","15.634","14.61","13.137","12.31461","15.56"
-"INTL.33-12-CSK-BKWH.A","        Former Czechoslovakia","4.8","4.2","3.7","3.9","3.2","4.3","4","4.853","4.355","4.229","3.919","3.119","3.602","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--"
-"INTL.33-12-SCG-BKWH.A","        Former Serbia and Montenegro","--","--","--","--","--","--","--","--","--","--","--","--","11.23","10.395","11.016","12.071","14.266","12.636","12.763","13.243","11.88","12.326","11.633","9.752","11.01","11.912","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--"
-"INTL.33-12-YUG-BKWH.A","        Former Yugoslavia","27.868","25.044","23.295","21.623","25.645","24.363","27.474","25.98","25.612","23.256","19.601","18.929","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--"
-"INTL.33-12-FRA-BKWH.A","        France","68.253","70.358","68.6","67.515","64.01","60.248","60.953","68.623","73.952","45.744","52.796","56.277","68.313","64.3","78.057","72.196","64.43","63.151","61.479","71.832","66.466","73.888","59.992","58.567","59.276","50.965","55.741","57.029","63.017","56.428","61.945","45.184","59.099","71.042","62.993","54.876","60.094","49.389","64.485","56.98242","64.84"
-"INTL.33-12-DEU-BKWH.A","        Germany","--","--","--","--","--","--","--","--","--","--","--","14.742","17.223","17.699","19.731","21.562","21.737","17.18343","17.044","19.451","21.515","22.506","22.893","19.071","20.866","19.442","19.808","20.957","20.239","18.841","20.678","17.323","21.331","22.66","19.31","18.664","20.214","19.985","17.815","19.86039","24.75"
-"INTL.33-12-DDR-BKWH.A","        Germany, East","1.658","1.718","1.748","1.683","1.748","1.758","1.767","1.726","1.719","1.551","1.389","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--"
-"INTL.33-12-DEUW-BKWH.A","        Germany, West","17.125","17.889","17.694","16.713","16.434","15.354","16.526","18.36","18.128","16.482","15.769","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--"
-"INTL.33-12-GIB-BKWH.A","        Gibraltar","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0"
-"INTL.33-12-GRC-BKWH.A","        Greece","3.396","3.398","3.551","2.331","2.852","2.792","3.222","2.768","2.354","1.888","1.751","3.068","2.181","2.26","2.573","3.494","4.305","3.84318","3.68","4.546","3.656","2.076","2.772","4.718","4.625","4.967","5.806","2.565","3.279","5.32","7.431","3.998","4.387","6.337","4.464","5.782","5.543","3.962","5.035","3.9798","3.43"
-"INTL.33-12-HUN-BKWH.A","        Hungary","0.111","0.166","0.158","0.153","0.179","0.153","0.152","0.167","0.167","0.156","0.176","0.192","0.156","0.164","0.159","0.161","0.205","0.21384","0.15345","0.179","0.176","0.184","0.192","0.169","0.203","0.2","0.184","0.208","0.211","0.226","0.184","0.216","0.206","0.208","0.294","0.227","0.253","0.214","0.216","0.21681","0.24"
-"INTL.33-12-ISL-BKWH.A","        Iceland","3.053","3.085","3.407","3.588","3.738","3.667","3.846","3.918","4.169","4.217","4.162","4.162","4.267","4.421","4.47","4.635","4.724","5.15493","5.565","5.987","6.292","6.512","6.907","7.017","7.063","6.949","7.22","8.31","12.303","12.156","12.51","12.382","12.214","12.747","12.554","13.541","13.092","13.892","13.679","13.32441","12.46"
-"INTL.33-12-IRL-BKWH.A","        Ireland","0.833","0.855","0.792","0.776","0.68","0.824","0.91","0.673","0.862","0.684","0.69","0.738","0.809","0.757","0.911","0.706","0.715","0.67122","0.907","0.838","0.838","0.59","0.903","0.592","0.624","0.625","0.717","0.66","0.959","0.893","0.593","0.699","0.795","0.593","0.701","0.798","0.674","0.685","0.687","0.87813","1.21"
-"INTL.33-12-ITA-BKWH.A","        Italy","44.997","42.782","41.216","40.96","41.923","40.616","40.626","39.05","40.205","33.647","31.31","41.817","41.778","41.011","44.212","37.404","41.617","41.18697","40.808","44.911","43.763","46.343","39.125","33.303","41.915","35.706","36.624","32.488","41.207","48.647","50.506","45.36477","41.45625","52.24626","57.95955","45.08163","42.00768","35.83701","48.29913","45.31824","47.72"
-"INTL.33-12-XKS-BKWH.A","        Kosovo","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","0.075","0.119","0.154","0.104","0.095","0.142","0.149","0.139","0.243","0.177","0.27027","0.2079","0.26"
-"INTL.33-12-LVA-BKWH.A","        Latvia","--","--","--","--","--","--","--","--","--","--","--","--","2.498","2.846","3.272","2.908","1.841","2.922","2.99","2.729","2.791","2.805","2.438","2.243","3.078","3.293","2.671","2.706","3.078","3.422","3.488","2.857","3.677","2.838","1.953","1.841","2.523","4.356","2.417","2.08692","2.59"
-"INTL.33-12-LTU-BKWH.A","        Lithuania","--","--","--","--","--","--","--","--","--","--","--","--","0.308","0.389","0.447","0.369","0.323","0.291","0.413","0.409","0.336","0.322","0.35","0.323","0.417","0.446193","0.393","0.417","0.398","0.42","0.535","0.475","0.419","0.516","0.395","0.346","0.45","0.597","0.427","0.34254","1.06"
-"INTL.33-12-LUX-BKWH.A","        Luxembourg","0.086","0.095","0.084","0.083","0.088","0.071","0.084","0.101","0.097","0.072","0.07","0.083","0.069","0.066","0.117","0.087","0.059","0.082","0.114","0.084","0.119","0.117","0.098","0.078","0.103","0.093","0.11","0.116","0.131","0.105","0.104","0.061","0.095","0.114","0.104","0.095","0.111","0.082","0.089","0.10593","1.09"
-"INTL.33-12-MLT-BKWH.A","        Malta","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0"
-"INTL.33-12-MNE-BKWH.A","        Montenegro","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","1.733","1.271","1.524","2.05","2.723","1.192","1.462","2.479","1.734","1.476","1.825","1.014","2.09187","1.78","1.8"
-"INTL.33-12-NLD-BKWH.A","        Netherlands","0","0","0","0","0","0.003","0.003","0.001","0.002","0.037","0.119","0.079","0.119","0.091","0.1","0.087","0.079","0.09108","0.111","0.089","0.141","0.116","0.109","0.071","0.094","0.087","0.105","0.106","0.101","0.097","0.105","0.057","0.104","0.114","0.112","0.093","0.1","0.061","0.072","0.07326","0.05"
-"INTL.33-12-MKD-BKWH.A","        North Macedonia","--","--","--","--","--","--","--","--","--","--","--","--","0.817","0.517","0.696","0.793","0.842","0.891","1.072","1.375","1.158","0.62","0.749","1.36","1.467","1.477","1.634","1","0.832","1.257","2.407","1.419","1.031","1.568","1.195","1.846","1.878","1.099","1.773","1.15236","1.24"
-"INTL.33-12-NOR-BKWH.A","        Norway","82.717","91.876","91.507","104.704","104.895","101.464","95.321","102.341","107.919","117.369","119.933","109.032","115.505","118.024","110.398","120.315","102.823","108.677","114.546","120.237","140.4","119.258","128.078","104.425","107.693","134.331","118.175","132.319","137.654","124.03","116.257","119.78","141.189","127.551","134.844","136.662","142.244","141.651","138.202","123.66288","141.69"
-"INTL.33-12-POL-BKWH.A","        Poland","2.326","2.116","1.528","1.658","1.394","1.833","1.534","1.644","1.775","1.593","1.403","1.411","1.492","1.473","1.716","1.868","1.912","1.941","2.286","2.133","2.085","2.302","2.256","1.654","2.06","2.179","2.022","2.328","2.13","2.351","2.9","2.313","2.02","2.421","2.165","1.814","2.117","2.552","1.949","1.93842","2.93"
-"INTL.33-12-PRT-BKWH.A","        Portugal","7.873","4.934","6.82","7.897","9.609","10.512","8.364","9.005","12.037","5.72","9.065","8.952","4.599","8.453","10.551","8.26","14.613","12.97395","12.853","7.213","11.21","13.894","7.722","15.566","9.77","4.684","10.892","9.991","6.73","8.201","15.954","11.423","5.589","13.652","15.471","8.615","15.608","5.79","12.316","8.6526","13.96"
-"INTL.33-12-ROU-BKWH.A","        Romania","12.506","12.605","11.731","9.934","11.208","11.772","10.688","11.084","13.479","12.497","10.87","14.107","11.583","12.64","12.916","16.526","15.597","17.334","18.69","18.107","14.63","14.774","15.886","13.126","16.348","20.005","18.172","15.806","17.023","15.379","19.684","14.581","11.945","14.807","18.618","16.467","17.848","14.349","17.48736","15.65289","15.53"
-"INTL.33-12-SRB-BKWH.A","        Serbia","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","10.855","9.937","9.468","10.436","11.772","8.58","9.193","10.101","10.893","9.979","10.684","9.061","10.53261","10.07028","9.66"
-"INTL.33-12-SVK-BKWH.A","        Slovakia","--","--","--","--","--","--","--","--","--","--","--","--","--","3.432","4.311","4.831","4.185","4.023","4.224","4.429","4.569","4.878","5.215","3.4452","4.059","4.592","4.355","4.406","4","4.324","5.184","3.211","3.687","4.329","3.762","3.701","4.302","4.321","3.506","4.27383","4.67"
-"INTL.33-12-SVN-BKWH.A","        Slovenia","--","--","--","--","--","--","--","--","--","--","--","--","3.379","2.974","3.348","3.187","3.616","3.046","3.4","3.684","3.771","3.741","3.265","2.916","4.033","3.426","3.555","3.233","3.978","4.666","4.452","3.506","3.841","4.562","6.011","3.75","4.443","3.814","4.643","4.43421","5.24"
-"INTL.33-12-ESP-BKWH.A","        Spain","29.16","21.64","25.99","26.696","31.088","30.895","26.105","27.016","34.76","19.046","25.16","27.01","18.731","24.133","27.898","22.881","39.404","34.43","33.665","22.634","29.274","40.617","22.691","40.643","31.359","18.209","25.699","27.036","23.13","26.147","41.576","30.07","20.192","36.45","38.815","27.656","35.77","18.007","33.743","24.23025","33.34"
-"INTL.33-12-SWE-BKWH.A","        Sweden","58.133","59.006","54.369","62.801","67.106","70.095","60.134","70.95","69.016","70.911","71.778","62.603","73.588","73.905","58.508","67.421","51.2226","68.365","74.25","70.974","77.798","78.269","65.696","53.005","59.522","72.075","61.106","65.497","68.378","65.193","66.279","66.047","78.333","60.81","63.227","74.734","61.645","64.651","61.79","64.46583","71.6"
-"INTL.33-12-CHE-BKWH.A","        Switzerland","32.481","35.13","35.974","35.069","29.871","31.731","32.576","34.328","35.437","29.477","29.497","31.756","32.373","35.416","38.678","34.817","28.458","33.70257","33.136","39.604","36.466","40.895","34.862","34.471","33.411","30.914","30.649","34.898","35.676","35.366","35.704","32.069","38.218","38.08","37.659","37.879","34.281","33.754","34.637","37.6596","40.62"
-"INTL.33-12-TUR-BKWH.A","        Turkey","11.159","12.308","13.81","11.13","13.19","11.822","11.637","18.314","28.447","17.61","22.917","22.456","26.302","33.611","30.28","35.186","40.07","39.41784","41.80671","34.33","30.57","23.77","33.346","34.977","45.623","39.165","43.802","35.492","32.937","35.598","51.423","51.155","56.669","58.225","39.75","65.856","66.686","57.824","59.49","87.99714","77.39"
-"INTL.33-12-GBR-BKWH.A","        United Kingdom","3.921","4.369","4.543","4.548","3.992","4.08","4.767","4.13","4.915","4.732","5.119","4.534","5.329","4.237","5.043","4.79","3.359","4.127","5.067","5.283","5.035","4.015","4.74","3.195","4.795","4.873","4.547","5.026","5.094","5.178","3.566","5.655","5.286","4.667","5.832","6.246","5.342","5.836","5.189","5.89941","7.64"
+https://www.eia.gov/international/data/world/electricity/electricity-generation?pd=2&p=000000000000000000000000000000g&u=1&f=A&v=mapbubble&a=-&i=none&vo=value&t=R&g=000000000000002&l=73-1028i008017kg6368g80a4k000e0ag00gg0004g8g0ho00g000400008&l=72-00000000000000000000000000080000000000000000000g&s=315532800000&e=1609459200000&ev=false&,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Report generated on: 01-06-2023 21:17:46,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+API,,1980,1981,1982,1983,1984,1985,1986,1987,1988,1989,1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021
+,hydroelectricity net generation (billion kWh),,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+INTL.33-12-EURO-BKWH.A,    Europe,"458,018","464,155","459,881","473,685","481,241","476,739","459,535","491,085","534,517","465,365","474,466","475,47","509,041","526,448","531,815","543,743","529,114164","543,845616","562,491501","566,861453","588,644662","584,806195","539,051405","503,7067","542,112443","542,974669","535,006084","538,449707","565,143111","561,761402","617,547148","540,926277","598,055253","629,44709","617,111295","613,079848","627,720566217","560,362524","616,5081462","606,5997419","644,1106599","628,1390143"
+INTL.33-12-ALB-BKWH.A,        Albania,"2,919","3,018","3,093","3,167","3,241","3,315","3,365","3,979","3,713","3,846","2,82","3,483","3,187","3,281","3,733","4,162","5,669","4,978","4,872","5,231","4,548","3,519","3,477","5,117","5,411","5,319","4,951","2,76","3,759","5,201","7,49133","4,09068","4,67775","6,88941","4,67676","5,83605","7,70418","4,47975","8,46648","5,15394","5,281","8,891943"
+INTL.33-12-AUT-BKWH.A,        Austria,"28,501","30,008","29,893","29,577","28,384","30,288","30,496","25,401","35,151","34,641","31,179","31,112","34,483","36,336","35,349","36,696","33,874","35,744","36,792","40,292","41,418","40,05","39,825","32,883","36,394","36,31","35,48","36,732","37,969","40,487","36,466","32,511","41,862","40,138","39,001","35,255","37,954","36,462","35,73","40,43655","41,9356096","38,75133"
+INTL.33-12-BEL-BKWH.A,        Belgium,"0,274","0,377","0,325","0,331","0,348","0,282","0,339","0,425","0,354","0,3","0,263","0,226","0,338","0,252","0,342","0,335","0,237","0,30195","0,38511","0,338","0,455","0,437","0,356","0,245","0,314","0,285","0,355","0,385","0,406","0,325","0,298","0,193","0,353","0,376","0,289","0,314","0,367","0,268","0,3135","0,302","0,2669","0,3933"
+INTL.33-12-BIH-BKWH.A,        Bosnia and Herzegovina,--,--,--,--,--,--,--,--,--,--,--,--,"3,374","2,343","3,424","3,607","5,104","4,608","4,511","5,477","5,043","5,129","5,215","4,456","5,919","5,938","5,798","3,961","4,818","6,177","7,946","4,343","4,173","7,164","5,876","5,495","5,585","3,7521","6,35382","6,02019","4,58","6,722"
+INTL.33-12-BGR-BKWH.A,        Bulgaria,"3,674","3,58","3,018","3,318","3,226","2,214","2,302","2,512","2,569","2,662","1,859","2,417","2,042","1,923","1,453","2,291","2,89","2,726","3,066","2,725","2,646","1,72","2,172","2,999","3,136","4,294","4,196","2,845","2,796","3,435","4,98168","2,84328","3,14622","3,99564","4,55598","5,59845","3,8412","2,79972","5,09553","2,929499","2,820398","4,819205"
+INTL.33-12-HRV-BKWH.A,        Croatia,--,--,--,--,--,--,--,--,--,--,--,--,"4,298","4,302","4,881","5,212","7,156","5,234","5,403","6,524","5,794","6,482","5,311","4,827","6,888","6,27","5,94","4,194","5,164","6,663","9,035","4,983","4,789","8,536","8,917","6,327","6,784","5,255","7,62399","5,87268","5,6624","7,1277"
+INTL.33-12-CYP-BKWH.A,        Cyprus,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+INTL.33-12-CZE-BKWH.A,        Czechia,--,--,--,--,--,--,--,--,--,--,--,--,--,"1,355","1,445","1,982","1,949","1,68201","1,382","1,664","1,7404","2,033","2,467","1,369","1,999","2,356","2,525","2,068","2,004","2,405","2,775","1,95","2,107","2,704","1,909","1,779","1,983","1,852","1,615","1,98792","2,143884","2,40852"
+INTL.33-12-DNK-BKWH.A,        Denmark,"0,03","0,031","0,028","0,036","0,028","0,027","0,029","0,029","0,032","0,027","0,027","0,026","0,028","0,027","0,033","0,03","0,019","0,019","0,02673","0,031","0,03","0,028","0,032","0,021","0,027","0,023","0,023","0,028","0,026","0,019","0,021","0,017","0,017","0,013","0,015","0,01803","0,01927","0,017871","0,0148621","0,0172171","0,017064","0,016295"
+INTL.33-12-EST-BKWH.A,        Estonia,--,--,--,--,--,--,--,--,--,--,--,--,"0,001","0,001","0,003","0,002","0,002","0,003","0,004","0,004","0,005","0,007","0,006","0,013","0,022","0,022","0,014","0,021","0,028","0,032","0,027","0,029999","0,042","0,026","0,027","0,027","0,035","0,025999","0,0150003","0,0189999","0,03","0,0248"
+INTL.33-12-FRO-BKWH.A,        Faroe Islands,"0,049","0,049","0,049","0,049","0,049","0,049","0,049","0,049","0,062","0,071","0,074","0,074","0,083","0,073","0,075","0,075","0,069564","0,075066","0,076501","0,069453","0,075262","0,075195","0,095535","0,08483","0,093443","0,097986","0,099934","0,103407","0,094921","0,091482","0,06676","0,092","0,099","0,091","0,121","0,132","0,105","0,11","0,107","0,102","0,11","0,11"
+INTL.33-12-FIN-BKWH.A,        Finland,"10,115","13,518","12,958","13,445","13,115","12,211","12,266","13,658","13,229","12,9","10,75","13,065","14,956","13,341","11,669","12,796","11,742","12,11958","14,9","12,652","14,513","13,073","10,668","9,495","14,919","13,646","11,379","14,035","16,941","12,559","12,743","12,278001","16,666998","12,672","13,240001","16,583999","15,634127","14,609473","13,1369998","12,2454823","15,883","15,766"
+INTL.33-12-CSK-BKWH.A,        Former Czechoslovakia,"4,8","4,2","3,7","3,9","3,2","4,3",4,"4,853","4,355","4,229","3,919","3,119","3,602",--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--
+INTL.33-12-SCG-BKWH.A,        Former Serbia and Montenegro,--,--,--,--,--,--,--,--,--,--,--,--,"11,23","10,395","11,016","12,071","14,266","12,636","12,763","13,243","11,88","12,326","11,633","9,752","11,01","11,912",--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--
+INTL.33-12-YUG-BKWH.A,        Former Yugoslavia,"27,868","25,044","23,295","21,623","25,645","24,363","27,474","25,98","25,612","23,256","19,601","18,929",--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--
+INTL.33-12-FRA-BKWH.A,        France,"68,253","70,358","68,6","67,515","64,01","60,248","60,953","68,623","73,952","45,744","52,796","56,277","68,313","64,3","78,057","72,196","64,43","63,151","61,479","71,832","66,466","73,888","59,992","58,567","59,276","50,965","55,741","57,029","63,017","56,428","61,945","45,184","59,099","71,042","62,993","54,876","60,094","49,389","64,485","56,913891","62,06191","58,856657"
+INTL.33-12-DEU-BKWH.A,        Germany,--,--,--,--,--,--,--,--,--,--,--,"14,742","17,223","17,699","19,731","21,562","21,737","17,18343","17,044","19,451","21,515","22,506","22,893","19,071","20,866","19,442","19,808","20,957","20,239","18,841","20,678","17,323","21,331","22,66","19,31","18,664","20,214","19,985","17,694","19,731","18,322","19,252"
+INTL.33-12-DDR-BKWH.A,"        Germany, East","1,658","1,718","1,748","1,683","1,748","1,758","1,767","1,726","1,719","1,551","1,389",--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--
+INTL.33-12-DEUW-BKWH.A,"        Germany, West","17,125","17,889","17,694","16,713","16,434","15,354","16,526","18,36","18,128","16,482","15,769",--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--
+INTL.33-12-GIB-BKWH.A,        Gibraltar,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+INTL.33-12-GRC-BKWH.A,        Greece,"3,396","3,398","3,551","2,331","2,852","2,792","3,222","2,768","2,354","1,888","1,751","3,068","2,181","2,26","2,573","3,494","4,305","3,84318","3,68","4,546","3,656","2,076","2,772","4,718","4,625","4,967","5,806","2,565","3,279","5,32","7,431","3,998","4,387","6,337","4,464","5,782","5,543","3,962","5,035","3,9798","3,343687","5,909225"
+INTL.33-12-HUN-BKWH.A,        Hungary,"0,111","0,166","0,158","0,153","0,179","0,153","0,152","0,167","0,167","0,156","0,176","0,192","0,156","0,164","0,159","0,161","0,205","0,21384","0,15345","0,179","0,176","0,184","0,192","0,169","0,203","0,2","0,184","0,208","0,211","0,226","0,184","0,215999","0,205999","0,207999","0,294001","0,226719","0,253308","0,213999","0,216","0,2129999","0,238","0,202379"
+INTL.33-12-ISL-BKWH.A,        Iceland,"3,053","3,085","3,407","3,588","3,738","3,667","3,846","3,918","4,169","4,217","4,162","4,162","4,267","4,421","4,47","4,635","4,724","5,15493","5,565","5,987","6,292","6,512","6,907","7,017","7,063","6,949","7,22","8,31","12,303","12,156","12,509999","12,381999","12,213999","12,747001","12,554","13,541","13,091609","13,891929","13,679377","13,32911","12,9196201","13,5746171"
+INTL.33-12-IRL-BKWH.A,        Ireland,"0,833","0,855","0,792","0,776","0,68","0,824","0,91","0,673","0,862","0,684","0,69","0,738","0,809","0,757","0,911","0,706","0,715","0,67122","0,907","0,838","0,838","0,59","0,903","0,592","0,624","0,625","0,717","0,66","0,959","0,893","0,593","0,699","0,795","0,593","0,701","0,798","0,674","0,685","0,687","0,87813","0,932656","0,750122"
+INTL.33-12-ITA-BKWH.A,        Italy,"44,997","42,782","41,216","40,96","41,923","40,616","40,626","39,05","40,205","33,647","31,31","41,817","41,778","41,011","44,212","37,404","41,617","41,18697","40,808","44,911","43,763","46,343","39,125","33,303","41,915","35,706","36,624","32,488","41,207","48,647","50,506","45,36477","41,45625","52,24626","57,95955","45,08163","42,00768","35,83701","48,29913","45,31824","47,551784","44,739"
+INTL.33-12-XKS-BKWH.A,        Kosovo,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,"0,075","0,119","0,154","0,104","0,095","0,142","0,149","0,139","0,243","0,177","0,27027","0,2079","0,262826","0,300635"
+INTL.33-12-LVA-BKWH.A,        Latvia,--,--,--,--,--,--,--,--,--,--,--,--,"2,498","2,846","3,272","2,908","1,841","2,922","2,99","2,729","2,791","2,805","2,438","2,243","3,078","3,293","2,671","2,706","3,078","3,422","3,487998","2,8568","3,677","2,838","1,953","1,841","2,522819","4,355513","2,4170639","2,0958919","2,5840101","2,6889293"
+INTL.33-12-LTU-BKWH.A,        Lithuania,--,--,--,--,--,--,--,--,--,--,--,--,"0,308","0,389","0,447","0,369","0,323","0,291","0,413","0,409","0,336","0,322","0,35","0,323","0,417","0,446193","0,393","0,417","0,398","0,42","0,535","0,475","0,419","0,516","0,395","0,346","0,45","0,597","0,427","0,34254","0,3006","0,3837"
+INTL.33-12-LUX-BKWH.A,        Luxembourg,"0,086","0,095","0,084","0,083","0,088","0,071","0,084","0,101","0,097","0,072","0,07","0,083","0,069","0,066","0,117","0,087","0,059","0,082","0,114","0,084","0,119","0,117","0,098","0,078","0,103","0,093","0,11","0,116","0,131","0,105","0,104","0,061","0,095","0,114","0,104","0,095","0,111","0,082","0,089","0,10593","0,091602","0,1068"
+INTL.33-12-MLT-BKWH.A,        Malta,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+INTL.33-12-MNE-BKWH.A,        Montenegro,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,"1,733","1,271","1,524","2,05","2,723","1,192","1,462","2,479","1,734","1,476","1,825","1,014","1,693443","1,262781","0,867637","1,212652"
+INTL.33-12-NLD-BKWH.A,        Netherlands,0,0,0,0,0,"0,003","0,003","0,001","0,002","0,037","0,119","0,079","0,119","0,091","0,1","0,087","0,079","0,09108","0,111","0,089","0,141","0,116","0,109","0,071","0,094","0,087","0,105","0,106","0,101","0,097","0,105","0,057","0,104389","0,11431","0,112202","0,0927","0,100078","0,060759","0,0723481","0,074182","0,0462851","0,0838927"
+INTL.33-12-MKD-BKWH.A,        North Macedonia,--,--,--,--,--,--,--,--,--,--,--,--,"0,817","0,517","0,696","0,793","0,842","0,891","1,072","1,375","1,158","0,62","0,749","1,36","1,467","1,477","1,634",1,"0,832","1,257","2,407","1,419","1,031","1,568","1,195","1,846","1,878","1,099","1,773","1,15236","1,277144","1,451623"
+INTL.33-12-NOR-BKWH.A,        Norway,"82,717","91,876","91,507","104,704","104,895","101,464","95,321","102,341","107,919","117,369","119,933","109,032","115,505","118,024","110,398","120,315","102,823","108,677","114,546","120,237","140,4","119,258","128,078","104,425","107,693","134,331","118,175","132,319","137,654","124,03","116,257","119,78","141,189","127,551","134,844","136,662","142,244","141,651","138,202","123,66288","141,69",144
+INTL.33-12-POL-BKWH.A,        Poland,"2,326","2,116","1,528","1,658","1,394","1,833","1,534","1,644","1,775","1,593","1,403","1,411","1,492","1,473","1,716","1,868","1,912","1,941","2,286","2,133","2,085","2,302","2,256","1,654","2,06","2,179","2,022","2,328","2,13","2,351","2,9","2,313","2,02","2,421","2,165","1,814","2,117","2,552","1,949","1,93842","2,118337","2,339192"
+INTL.33-12-PRT-BKWH.A,        Portugal,"7,873","4,934","6,82","7,897","9,609","10,512","8,364","9,005","12,037","5,72","9,065","8,952","4,599","8,453","10,551","8,26","14,613","12,97395","12,853","7,213","11,21","13,894","7,722","15,566","9,77","4,684","10,892","9,991","6,73","8,201","15,954","11,423","5,589","13,652","15,471","8,615","15,608","5,79","12,316","8,6526","12,082581","11,846464"
+INTL.33-12-ROU-BKWH.A,        Romania,"12,506","12,605","11,731","9,934","11,208","11,772","10,688","11,084","13,479","12,497","10,87","14,107","11,583","12,64","12,916","16,526","15,597","17,334","18,69","18,107","14,63","14,774","15,886","13,126","16,348","20,005","18,172","15,806","17,023","15,379","19,684","14,581","11,945","14,807","18,618","16,467","17,848","14,349","17,48736","15,580622","15,381243","17,376933"
+INTL.33-12-SRB-BKWH.A,        Serbia,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,--,"10,855","9,937","9,468","10,436","11,772","8,58","9,193","10,101","10,893","9,979","10,684","9,061","10,53261","9,457175","9,034496","11,284232"
+INTL.33-12-SVK-BKWH.A,        Slovakia,--,--,--,--,--,--,--,--,--,--,--,--,--,"3,432","4,311","4,831","4,185","4,023","4,224","4,429","4,569","4,878","5,215","3,4452","4,059","4,592","4,355","4,406",4,"4,324","5,184","3,211","3,687","4,329","3,762","3,701","4,302","4,321","3,506","4,27383","4,517","4,17"
+INTL.33-12-SVN-BKWH.A,        Slovenia,--,--,--,--,--,--,--,--,--,--,--,--,"3,379","2,974","3,348","3,187","3,616","3,046","3,4","3,684","3,771","3,741","3,265","2,916","4,033","3,426","3,555","3,233","3,978","4,666","4,452","3,506","3,841","4,562","6,011","3,75","4,443","3,814","4,643","4,43421","4,93406","4,711944"
+INTL.33-12-ESP-BKWH.A,        Spain,"29,16","21,64","25,99","26,696","31,088","30,895","26,105","27,016","34,76","19,046","25,16","27,01","18,731","24,133","27,898","22,881","39,404","34,43","33,665","22,634","29,274","40,617","22,691","40,643","31,359","18,209","25,699","27,036","23,13","26,147","41,576","30,07","20,192","36,45","38,815","27,656","35,77","18,007","33,743","24,23025","30,507","29,626"
+INTL.33-12-SWE-BKWH.A,        Sweden,"58,133","59,006","54,369","62,801","67,106","70,095","60,134","70,95","69,016","70,911","71,778","62,603","73,588","73,905","58,508","67,421","51,2226","68,365","74,25","70,974","77,798","78,269","65,696","53,005","59,522","72,075","61,106","65,497","68,378","65,193","66,279","66,047","78,333","60,81","63,227","74,734","61,645","64,651","61,79","64,46583","71,6","71,086"
+INTL.33-12-CHE-BKWH.A,        Switzerland,"32,481","35,13","35,974","35,069","29,871","31,731","32,576","34,328","35,437","29,477","29,497","31,756","32,373","35,416","38,678","34,817","28,458","33,70257","33,136","37,104","33,854","38,29","32,323","31,948","30,938","28,664","28,273","32,362","33,214","32,833","33,261","29,906","35,783","35,628","35,122","35,378","31,984","31,47968","32,095881","35,156989","37,867647","36,964485"
+INTL.33-12-TUR-BKWH.A,        Turkey,"11,159","12,308","13,81","11,13","13,19","11,822","11,637","18,314","28,447","17,61","22,917","22,456","26,302","33,611","30,28","35,186","40,07","39,41784","41,80671","34,33","30,57","23,77","33,346","34,977","45,623","39,165","43,802","35,492","32,937","35,598","51,423001","51,154999","56,668998","58,225","39,750001","65,856","66,685883","57,823851","59,490211","88,2094218","78,094369","55,1755392"
+INTL.33-12-GBR-BKWH.A,        United Kingdom,"3,921","4,369","4,543","4,548","3,992","4,08","4,767","4,13","4,915","4,732","5,119","4,534","5,329","4,237","5,043","4,79","3,359","4,127","5,117","5,336","5,085","4,055","4,78787","3,22767","4,844","4,92149","4,59315","5,0773","5,14119","5,22792","3,59138","5,69175","5,30965","4,70147","5,8878","6,29727","5,370412217","5,88187","5,44327","5,84628","6,75391","5,0149"
+,    Eurasia,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+INTL.33-12-MDA-BKWH.A,        Moldova,--,--,--,--,--,--,--,--,--,--,--,--,"0,255","0,371","0,275","0,321","0,362","0,378","0,387","0,363","0,392","0,359","0,348","0,358","0,35","0,359","0,365","0,354","0,385","0,354","0,403","0,348","0,266","0,311","0,317","0,265","0,228","0,282","0,27324","0,29799","0,276","0,316"
+INTL.33-12-UKR-BKWH.A,        Ukraine,--,--,--,--,--,--,--,--,--,--,--,--,"7,725","10,929","11,997","9,853","8,546","9,757","15,756","14,177","11,161","11,912","9,531","9,146","11,635","12,239","12,757","10,042","11,397","11,817","13,02","10,837","10,374","13,663","8,393","5,343","7,594","8,856","10,32372","6,5083","7,5638","10,3326"
--- a/data/switzerland-new_format-all_years.csv
+++ b/data/switzerland-new_format-all_years.csv
@ -0,0 +1,25 @@
+country,item,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022
+CH,total residential,268.2,223.4,243.4,261.3,214.2,229.1,241.2,236.5,223.7,226.5,219.1,241.2,211.3
+CH,total residential space,192.2,149,168.1,185.5,139.7,154.4,167.3,161.5,147.2,150.4,140.2,166.2,131.9
+CH,total residential water,32.2,31.6,31.9,32.2,31.7,31.9,31.8,31.8,31.8,31.7,33.3,32.5,32.5
+CH,total residential cooking,9.3,9.3,9.3,9.4,9.5,9.6,9.9,10,10.1,10.2,10.5,10.3,10.3
+CH,electricity residential,67.9,63.7,65.7,67.6,63,64.4,69.7,69.2,67.7,68.1,68.7,70.8,66.8
+CH,electricity residential space,15.9,12.8,14.3,15.8,12.3,13.5,15.8,15.6,14.7,15.3,14.8,17.8,14.8
+CH,electricity residential water,8.8,8.5,8.5,8.6,8.5,8.6,8.9,9,9.2,9.3,9.7,9.5,9.5
+CH,electricity residential cooking,4.9,4.9,4.9,4.9,5,5,5,5.1,5.1,5.1,5.4,5.2,5.3
+CH,total services,145.9,127.4,136.7,144,124.5,132.5,150.5,147.7,141.5,143.1,129.7,144.2,122.5
+CH,total services space,80,62.2,70.8,77.4,58.3,64.3,77,74.4,68.2,69.8,64.3,75.7,58.7
+CH,total services water,10.1,10,10.1,10.1,10,10,11.4,11.3,11.2,11.1,9.7,10.4,12
+CH,total services cooking,2.5,2.4,2.3,2.3,2.4,2.3,3.1,3.1,3.2,3.3,2.1,2.6,3.2
+CH,electricity services,60.5,59.2,60.3,61.4,60.3,62.6,65.9,65.7,65.5,65.6,58.8,61.6,61.6
+CH,electricity services space,4,3.2,3.8,4.2,3.3,3.6,2.7,2.5,2.3,2.3,2.2,2.5,2.5
+CH,electricity services water,0.7,0.7,0.7,0.7,0.7,0.7,1.2,1.1,1.1,1.1,0.9,1,1
+CH,electricity services cooking,2.5,2.4,2.3,2.3,2.4,2.3,3.1,3.1,3.1,3.2,3.3,2.1,3.2
+CH,total rail,11.5,11.1,11.2,11.4,11.1,11.4,11.6,11.4,11.2,11,10.2,10.6,10.8
+CH,total road,199.4,200.4,200.4,201.2,202,203.1,203.9,203.7,202.6,200.5,182.6,188.3,193.3
+CH,electricity road,0,0,0,0,0,0,0.1,0.2,0.3,0.4,0.5,0.8,1.3
+CH,electricity rail,11.5,11.1,11.2,11.4,11.1,11.4,11.5,11.3,11.1,11,10.1,10.6,10.7
+CH,total domestic aviation,3.3,3.2,3.4,3.4,3.5,3.5,3.6,3.1,3.1,2.9,2.5,2.8,3
+CH,total international aviation,58,62,63.5,64.2,64.5,66.8,70.6,72.8,77.2,78.2,28.2,31.2,56.8
+CH,total domestic navigation,1.6,1.6,1.6,1.6,1.6,1.6,1.4,1.4,1.4,1.4,1.4,1.4,1.4
+CH,total international navigation,0,0,0,0,0,0,0,0,0,0,0,0,0
--- a/doc/Makefile
+++ b/doc/Makefile
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT

--- a/doc/conf.py
+++ b/doc/conf.py
@ -72,7 +72,7 @@ master_doc = "index"

 # General information about the project.
 project = "PyPSA-Eur"
-copyright = "2017-2023 Tom Brown (KIT, TUB, FIAS), Jonas Hoersch (KIT, FIAS), Fabian Hofmann (TUB, FIAS), Fabian Neumann (TUB, KIT), Marta Victoria (Aarhus University), Lisa Zeyen (KIT, TUB)"
+copyright = "2017-2024 Tom Brown (KIT, TUB, FIAS), Jonas Hoersch (KIT, FIAS), Fabian Hofmann (TUB, FIAS), Fabian Neumann (TUB, KIT), Marta Victoria (Aarhus University), Lisa Zeyen (KIT, TUB)"
 author = "Tom Brown (KIT, TUB, FIAS), Jonas Hoersch (KIT, FIAS), Fabian Hofmann (TUB, FIAS), Fabian Neumann (TUB, KIT), Marta Victoria (Aarhus University), Lisa Zeyen (KIT, TUB)"

 # The version info for the project you're documenting, acts as replacement for
@ -80,9 +80,9 @@ author = "Tom Brown (KIT, TUB, FIAS), Jonas Hoersch (KIT, FIAS), Fabian Hofmann
 # built documents.
 #
 # The short X.Y version.
-version = "0.8"
+version = "0.10"
 # The full version, including alpha/beta/rc tags.
-release = "0.8.1"
+release = "0.10.0"

 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
--- a/doc/configtables/adjustments.csv
+++ b/doc/configtables/adjustments.csv
@ -0,0 +1,8 @@
+,Unit,Values,Description
+adjustments,,,
+-- electricity,bool or dict,,"Parameter adjustments for capital cost, marginal cost, and maximum capacities of carriers. Applied in :mod:`prepare_network.`"
+-- -- {attr},,,"Attribute can be ``e_nom_opt``, ``p_nom_opt``, ``marginal_cost`` or ``capital_cost``"
+-- -- -- {carrier},float,per-unit,"Any carrier of the network to which parameter adjustment factor should be applied."
+-- sector,bool or dict,,"Parameter adjustments for capital cost, marginal cost, and maximum capacities of carriers. Applied in :mod:`prepare_sector_network.`"
+-- -- {attr},,,"Attribute can be ``e_nom_opt``, ``p_nom_opt``, ``marginal_cost`` or ``capital_cost``"
+-- -- -- {carrier},float,per-unit,"Any carrier of the network to which parameter adjustment factor should be applied."
--- a/doc/configtables/clustering.csv
+++ b/doc/configtables/clustering.csv
@ -1,4 +1,5 @@
 ,Unit,Values,Description
+focus_weights,,,Optionally specify the focus weights for the clustering of countries. For instance: `DE: 0.8` will distribute 80% of all nodes to Germany and 20% to the rest of the countries.
 simplify_network,,,
 -- to_substations,bool,"{'true','false'}","Aggregates all nodes without power injection (positive or negative, i.e. demand or generation) to electrically closest ones"
 -- algorithm,str,"One of {‘kmeans’, ‘hac’, ‘modularity‘}",
@ -16,3 +17,6 @@ aggregation_strategies,,,
 -- -- {key},str,"{key} can be any of the component of the generator (str). It’s value can be any that can be converted to pandas.Series using getattr(). For example one of {min, max, sum}.","Aggregates the component according to the given strategy. For example, if sum, then all values within each cluster are summed to represent the new generator."
 -- buses,,,
 -- -- {key},str,"{key} can be any of the component of the bus (str). It’s value can be any that can be converted to pandas.Series using getattr(). For example one of {min, max, sum}.","Aggregates the component according to the given strategy. For example, if sum, then all values within each cluster are summed to represent the new bus."
+temporal,,,Options for temporal resolution
+-- resolution_elec,--,"{false,``nH``; i.e. ``2H``-``6H``}","Resample the time-resolution by averaging over every ``n`` snapshots in :mod:`prepare_network`. **Warning:** This option should currently only be used with electricity-only networks, not for sector-coupled networks."
+-- resolution_sector,--,"{false,``nH``; i.e. ``2H``-``6H``}","Resample the time-resolution by averaging over every ``n`` snapshots in :mod:`prepare_sector_network`."
--- a/doc/configtables/costs.csv
+++ b/doc/configtables/costs.csv
@ -1,9 +1,12 @@
-,Unit,Values,Description
-year,--,"YYYY; e.g. '2030'","Year for which to retrieve cost assumptions of ``resources/costs.csv``."
-version,--,"vX.X.X; e.g. 'v0.5.0'","Version of ``technology-data`` repository to use."
-rooftop_share,--,float,"Share of rooftop PV when calculating capital cost of solar (joint rooftop and utility-scale PV)."
-fill_values,--,float,"Default values if not specified for a technology in ``resources/costs.csv``."
-capital_cost,EUR/MW,"Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.","For the given technologies, assumptions about their capital investment costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``."
-marginal_cost,EUR/MWh,"Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.","For the given technologies, assumptions about their marginal operating costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``."
-emission_prices,,,"Specify exogenous prices for emission types listed in ``network.carriers`` to marginal costs."
-- co2,EUR/t,float,"Exogenous price of carbon-dioxide added to the marginal costs of fossil-fuelled generators according to their carbon intensity. Added through the keyword ``Ep`` in the ``{opts}`` wildcard only in the rule :mod:`prepare_network``."
+,Unit,Values,Description
+year,--,YYYY; e.g. '2030',Year for which to retrieve cost assumptions of ``resources/costs.csv``.
+version,--,vX.X.X; e.g. 'v0.5.0',Version of ``technology-data`` repository to use.
+rooftop_share,--,float,Share of rooftop PV when calculating capital cost of solar (joint rooftop and utility-scale PV).
+social_discountrate,p.u.,float,Social discount rate to compare costs in different investment periods. 0.02 corresponds to a social discount rate of 2%.
+fill_values,--,float,Default values if not specified for a technology in ``resources/costs.csv``.
+capital_cost,EUR/MW,Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.,"For the given technologies, assumptions about their capital investment costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``."
+marginal_cost,EUR/MWh,Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.,"For the given technologies, assumptions about their marginal operating costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``."
+emission_prices,,,Specify exogenous prices for emission types listed in ``network.carriers`` to marginal costs.
+-- enable,bool,true or false,Add cost for a carbon-dioxide price configured in ``costs: emission_prices: co2`` to ``marginal_cost`` of generators (other emission types listed in ``network.carriers`` possible as well)
+-- co2,EUR/t,float,Exogenous price of carbon-dioxide added to the marginal costs of fossil-fuelled generators according to their carbon intensity. Added through the keyword ``Ep`` in the ``{opts}`` wildcard only in the rule :mod:`prepare_network``.
+-- co2_monthly_price,bool,true or false,Add monthly cost for a carbon-dioxide price based on historical values built by the rule ``build_monthly_prices``
--- a/doc/configtables/electricity.csv
+++ b/doc/configtables/electricity.csv
@ -1,6 +1,8 @@
 ,Unit,Values,Description
 voltages,kV,"Any subset of {220., 300., 380.}",Voltage levels to consider
+gaslimit_enable,bool,true or false,Add an overall absolute gas limit configured in ``electricity: gaslimit``.
 gaslimit,MWhth,float or false,Global gas usage limit
+co2limit_enable,bool,true or false,Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``.
 co2limit,:math:`t_{CO_2-eq}/a`,float,Cap on total annual system carbon dioxide emissions
 co2base,:math:`t_{CO_2-eq}/a`,float,Reference value of total annual system carbon dioxide emissions if relative emission reduction target is specified in ``{opts}`` wildcard.
 agg_p_nom_limits,file,path,Reference to ``.csv`` file specifying per carrier generator nominal capacity constraints for individual countries if ``'CCL'`` is in ``{opts}`` wildcard. Defaults to ``data/agg_p_nom_minmax.csv``.
@ -22,6 +24,8 @@ powerplants_filter,--,"use `pandas.query <https://pandas.pydata.org/pandas-docs/
 ,,,
 custom_powerplants,--,"use `pandas.query <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.query.html>`_ strings here, e.g. ``Country in ['Germany']``",Filter query for the custom powerplant database.
 ,,,
+everywhere_powerplants,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to add to every node in the model with zero initial capacity. To be used in combination with ``extendable_carriers`` to allow for building conventional powerplants irrespective of existing locations."
+,,,
 conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to include in the model from ``resources/powerplants.csv``. If an included carrier is also listed in ``extendable_carriers``, the capacity is taken as a lower bound."
 ,,,
 renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, hydro}",List of renewable generators to include in the model.
@ -34,3 +38,6 @@ estimate_renewable_capacities,,,
 -- -- Offshore,--,"Any subset of {offwind-ac, offwind-dc}","List of PyPSA-Eur carriers that is considered as (IRENA, OPSD) onshore technology."
 -- -- Offshore,--,{onwind},"List of PyPSA-Eur carriers that is considered as (IRENA, OPSD) offshore technology."
 -- -- PV,--,{solar},"List of PyPSA-Eur carriers that is considered as (IRENA, OPSD) PV technology."
+autarky,,,
+-- enable,bool,true or false,Require each node to be autarkic by removing all lines and links.
+-- by_country,bool,true or false,Require each country to be autarkic by removing all cross-border lines and links. ``electricity: autarky`` must be enabled.
--- a/doc/configtables/energy.csv
+++ b/doc/configtables/energy.csv
@ -1,7 +1,4 @@
 ,Unit,Values,Description
 energy_totals_year ,--,"{1990,1995,2000,2005,2010,2011,…} ",The year for the sector energy use. The year must be avaliable in the Eurostat report
 base_emissions_year ,--,"YYYY; e.g. 1990","The base year for the sector emissions. See `European Environment Agency (EEA) <https://www.eea.europa.eu/data-and-maps/data/national-emissions-reported-to-the-unfccc-and-to-the-eu-greenhouse-gas-monitoring-mechanism-16>`_."
-
-eurostat_report_year ,--,"{2016,2017,2018}","The publication year of the Eurostat report. 2016 includes Bosnia and Herzegovina, 2017 does not"
-
 emissions ,--,"{CO2, All greenhouse gases - (CO2 equivalent)}","Specify which sectoral emissions are taken into account. Data derived from EEA. Currently only CO2 is implemented."
--- a/doc/configtables/existing_capacities.csv
+++ b/doc/configtables/existing_capacities.csv
@ -3,4 +3,5 @@ grouping_years_power ,--,A list of years,Intervals to group existing capacities
 grouping_years_heat ,--,A list of years below 2020,Intervals to group existing capacities for heat

 threshold_capacity ,MW,float,Capacities generators and links of below threshold are removed during add_existing_capacities
+default_heating_lifetime ,years,int,Default lifetime for heating technologies
 conventional_carriers ,--,"Any subset of {uranium, coal, lignite, oil} ",List of conventional power plants to include in the sectoral network
--- a/doc/configtables/industry.csv
+++ b/doc/configtables/industry.csv
@ -17,6 +17,8 @@ HVC_primary_fraction,--,float,The fraction of high value chemicals (HVC) produce
 HVC_mechanical_recycling _fraction,--,float,The fraction of high value chemicals (HVC) produced using mechanical recycling
 HVC_chemical_recycling _fraction,--,float,The fraction of high value chemicals (HVC) produced using chemical recycling
 ,,,
+sector_ratios_fraction_future,--,Dictionary with planning horizons as keys.,The fraction of total progress in fuel and process switching achieved in the industry sector.
+basic_chemicals_without_NH3_production_today,Mt/a,float,"The amount of basic chemicals produced without ammonia (= 86 Mtethylene-equiv - 17 MtNH3)."
 HVC_production_today,MtHVC/a,float,"The amount of high value chemicals (HVC) produced. This includes ethylene, propylene and BTX. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, Figure 16, page 107"
 Mwh_elec_per_tHVC _mechanical_recycling,MWh/tHVC,float,"The energy amount of electricity needed to produce a ton of high value chemical (HVC) using mechanical recycling. From SI of `Meys et al (2020) <https://doi.org/10.1016/j.resconrec.2020.105010>`_, Table S5, for HDPE, PP, PS, PET. LDPE would be 0.756."
 Mwh_elec_per_tHVC _chemical_recycling,MWh/tHVC,float,"The energy amount of electricity needed to produce a ton of high value chemical (HVC) using chemical recycling. The default value is based on pyrolysis and electric steam cracking. From `Material Economics (2019) <https://materialeconomics.com/latest-updates/industrial-transformation-2050>`_, page 125"
--- a/doc/configtables/licenses-sector.csv
+++ b/doc/configtables/licenses-sector.csv
@ -9,9 +9,8 @@ Swiss energy statistics from Swiss Federal Office of Energy,switzerland-sfoe/,un
 BASt emobility statistics,emobility/,unknown,http://www.bast.de/DE/Verkehrstechnik/Fachthemen/v2-verkehrszaehlung/Stundenwerte.html?nn=626916
 BDEW heating profile,heat_load_profile_BDEW.csv,unknown,https://github.com/oemof/demandlib
 heating profiles for Aarhus,heat_load_profile_DK_AdamJensen.csv,unknown,Adam Jensen MA thesis at Aarhus University
-George Lavidas wind/wave costs,WindWaveWEC_GLTB.xlsx,unknown,George Lavidas
 co2 budgets,co2_budget.csv,CC BY 4.0,https://arxiv.org/abs/2004.11009
-existing heating potentials,existing_infrastructure/existing_heating_raw.csv,unknown,https://ec.europa.eu/energy/studies/mapping-and-analyses-current-and-future-2020-2030-heatingcooling-fuel-deployment_en?redir=1
+existing heating potentials,existing_infrastructure/existing_heating_raw.csv,unknown,https://energy.ec.europa.eu/publications/mapping-and-analyses-current-and-future-2020-2030-heatingcooling-fuel-deployment-fossilrenewables-1_en
 IRENA existing VRE capacities,existing_infrastructure/{solar|onwind|offwind}_capcity_IRENA.csv,unknown,https://www.irena.org/Statistics/Download-Data
 USGS ammonia production,myb1-2017-nitro.xls,unknown,https://www.usgs.gov/centers/nmic/nitrogen-statistics-and-information
 hydrogen salt cavern potentials,h2_salt_caverns_GWh_per_sqkm.geojson,CC BY 4.0,https://doi.org/10.1016/j.ijhydene.2019.12.161 https://doi.org/10.20944/preprints201910.0187.v1
--- a/doc/configtables/licenses.csv
+++ b/doc/configtables/licenses.csv
@ -5,7 +5,7 @@
 "naturalearth/*",,,,,http://www.naturalearthdata.com/about/terms-of-use/
 "NUTS_2013 _60M_SH/*","x","x",,"x",https://ec.europa.eu/eurostat/web/gisco/geodata/reference-data/administrative-units-statistical-units
 "cantons.csv","x",,"x",,https://en.wikipedia.org/wiki/Data_codes_for_Switzerland
-"EIA_hydro_generation _2000_2014.csv","x",,,,https://www.eia.gov/about/copyrights_reuse.php
+"eia_hydro_annual_generation.csv","x",,,,https://www.eia.gov/about/copyrights_reuse.php
 "GEBCO_2014_2D.nc","x",,,,https://www.gebco.net/data_and_products/gridded_bathymetry_data/documents/gebco_2014_historic.pdf
 "hydro_capacities.csv","x",,,,
 "je-e-21.03.02.xls","x","x",,,https://www.bfs.admin.ch/bfs/en/home/fso/swiss-federal-statistical-office/terms-of-use.html
--- a/doc/configtables/lines.csv
+++ b/doc/configtables/lines.csv
@ -5,6 +5,7 @@ s_nom_max,MW,"float","Global upper limit for the maximum capacity of each extend
 max_extension,MW,"float","Upper limit for the extended capacity of each extendable line."
 length_factor,--,float,"Correction factor to account for the fact that buses are *not* connected by lines through air-line distance."
 under_construction,--,"One of {'zero': set capacity to zero, 'remove': remove completely, 'keep': keep with full capacity}","Specifies how to handle lines which are currently under construction."
+reconnect_crimea,--,"true or false","Whether to reconnect Crimea to the Ukrainian grid"
 dynamic_line_rating,,,
 -- activate,bool,"true or false","Whether to take dynamic line rating into account"
 -- cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored."
--- a/doc/configtables/load.csv
+++ b/doc/configtables/load.csv
@ -1,5 +1,4 @@
 ,Unit,Values,Description
-power_statistics,bool,"{true, false}",Whether to load the electricity consumption data of the ENTSOE power statistics (only for files from 2019 and before) or from the ENTSOE transparency data (only has load data from 2015 onwards).
 interpolate_limit,hours,integer,"Maximum gap size (consecutive nans) which interpolated linearly."
 time_shift_for_large_gaps,string,string,"Periods which are used for copying time-slices in order to fill large gaps of nans. Have to be valid ``pandas`` period strings."
 manual_adjustments,bool,"{true, false}","Whether to adjust the load data manually according to the function in :func:`manual_adjustment`."
--- a/doc/configtables/offwind-ac.csv
+++ b/doc/configtables/offwind-ac.csv
@ -2,15 +2,15 @@
 cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored."
 resource,,,
 -- method,--,"Must be 'wind'","A superordinate technology type."
-- turbine,--,"One of turbine types included in `atlite <https://github.com/PyPSA/atlite/tree/master/atlite/resources/windturbine>`_","Specifies the turbine type and its characteristic power curve."
+-- turbine,--,"One of turbine types included in `atlite <https://github.com/PyPSA/atlite/tree/master/atlite/resources/windturbine>`_.  Can be a string or a dictionary with years as keys which denote the year another turbine model becomes available.","Specifies the turbine type and its characteristic power curve."
 capacity_per_sqkm,:math:`MW/km^2`,float,"Allowable density of wind turbine placement."
 correction_factor,--,float,"Correction factor for capacity factor time series."
 excluder_resolution,m,float,"Resolution on which to perform geographical elibility analysis."
 corine,--,"Any *realistic* subset of the `CORINE Land Cover code list <http://www.eea.europa.eu/data-and-maps/data/corine-land-cover-2006-raster-1/corine-land-cover-classes-and/clc_legend.csv/at_download/file>`_","Specifies areas according to CORINE Land Cover codes which are generally eligible for AC-connected offshore wind turbine placement."
+luisa,--,"Any subset of the `LUISA Base Map codes in Annex 1 <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_","Specifies areas according to the LUISA Base Map codes which are generally eligible for AC-connected offshore wind turbine placement."
 natura,bool,"{true, false}","Switch to exclude `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas. Area is excluded if ``true``."
 ship_threshold,--,float,"Ship density threshold from which areas are excluded."
 max_depth,m,float,"Maximum sea water depth at which wind turbines can be build. Maritime areas with deeper waters are excluded in the process of calculating the AC-connected offshore wind potential."
 min_shore_distance,m,float,"Minimum distance to the shore below which wind turbines cannot be build. Such areas close to the shore are excluded in the process of calculating the AC-connected offshore wind potential."
 max_shore_distance,m,float,"Maximum distance to the shore above which wind turbines cannot be build. Such areas close to the shore are excluded in the process of calculating the AC-connected offshore wind potential."
-potential,--,"One of {'simple', 'conservative'}","Method to compute the maximal installable potential for a node; confer :ref:`renewableprofiles`"
 clip_p_max_pu,p.u.,float,"To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero."
--- a/doc/configtables/offwind-dc.csv
+++ b/doc/configtables/offwind-dc.csv
@ -2,15 +2,15 @@
 cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored."
 resource,,,
 -- method,--,"Must be 'wind'","A superordinate technology type."
-- turbine,--,"One of turbine types included in `atlite <https://github.com/PyPSA/atlite/tree/master/atlite/resources/windturbine>`__","Specifies the turbine type and its characteristic power curve."
+-- turbine,--,"One of turbine types included in `atlite <https://github.com/PyPSA/atlite/tree/master/atlite/resources/windturbine>`_. Can be a string or a dictionary with years as keys which denote the year another turbine model becomes available.","Specifies the turbine type and its characteristic power curve."
 capacity_per_sqkm,:math:`MW/km^2`,float,"Allowable density of wind turbine placement."
 correction_factor,--,float,"Correction factor for capacity factor time series."
 excluder_resolution,m,float,"Resolution on which to perform geographical elibility analysis."
 corine,--,"Any *realistic* subset of the `CORINE Land Cover code list <http://www.eea.europa.eu/data-and-maps/data/corine-land-cover-2006-raster-1/corine-land-cover-classes-and/clc_legend.csv/at_download/file>`_","Specifies areas according to CORINE Land Cover codes which are generally eligible for AC-connected offshore wind turbine placement."
+luisa,--,"Any subset of the `LUISA Base Map codes in Annex 1 <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_","Specifies areas according to the LUISA Base Map codes which are generally eligible for DC-connected offshore wind turbine placement."
 natura,bool,"{true, false}","Switch to exclude `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas. Area is excluded if ``true``."
 ship_threshold,--,float,"Ship density threshold from which areas are excluded."
 max_depth,m,float,"Maximum sea water depth at which wind turbines can be build. Maritime areas with deeper waters are excluded in the process of calculating the AC-connected offshore wind potential."
 min_shore_distance,m,float,"Minimum distance to the shore below which wind turbines cannot be build."
 max_shore_distance,m,float,"Maximum distance to the shore above which wind turbines cannot be build."
-potential,--,"One of {'simple', 'conservative'}","Method to compute the maximal installable potential for a node; confer :ref:`renewableprofiles`"
 clip_p_max_pu,p.u.,float,"To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero."
--- a/doc/configtables/onwind.csv
+++ b/doc/configtables/onwind.csv
@ -2,14 +2,17 @@
 cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored."
 resource,,,
 -- method,--,"Must be 'wind'","A superordinate technology type."
-- turbine,--,"One of turbine types included in `atlite <https://github.com/PyPSA/atlite/tree/master/atlite/resources/windturbine>`__","Specifies the turbine type and its characteristic power curve."
+-- turbine,--,"One of turbine types included in `atlite <https://github.com/PyPSA/atlite/tree/master/atlite/resources/windturbine>`_. Can be a string or a dictionary with years as keys which denote the year another turbine model becomes available.","Specifies the turbine type and its characteristic power curve."
 capacity_per_sqkm,:math:`MW/km^2`,float,"Allowable density of wind turbine placement."
 corine,,,
 -- grid_codes,--,"Any subset of the `CORINE Land Cover code list <http://www.eea.europa.eu/data-and-maps/data/corine-land-cover-2006-raster-1/corine-land-cover-classes-and/clc_legend.csv/at_download/file>`_","Specifies areas according to CORINE Land Cover codes which are generally eligible for wind turbine placement."
 -- distance,m,float,"Distance to keep from areas specified in ``distance_grid_codes``"
 -- distance_grid_codes,--,"Any subset of the `CORINE Land Cover code list <http://www.eea.europa.eu/data-and-maps/data/corine-land-cover-2006-raster-1/corine-land-cover-classes-and/clc_legend.csv/at_download/file>`_","Specifies areas according to CORINE Land Cover codes to which wind turbines must maintain a distance specified in the setting ``distance``."
+luisa,,,
+-- grid_codes,--,"Any subset of the `LUISA Base Map codes in Annex 1 <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_","Specifies areas according to the LUISA Base Map codes which are generally eligible for wind turbine placement."
+-- distance,m,float,"Distance to keep from areas specified in ``distance_grid_codes``"
+-- distance_grid_codes,--,"Any subset of the `LUISA Base Map codes in Annex 1 <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_","Specifies areas according to the LUISA Base Map codes to which wind turbines must maintain a distance specified in the setting ``distance``."
 natura,bool,"{true, false}","Switch to exclude `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas. Area is excluded if ``true``."
-potential,--,"One of {'simple', 'conservative'}","Method to compute the maximal installable potential for a node; confer :ref:`renewableprofiles`"
 clip_p_max_pu,p.u.,float,"To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero."
 correction_factor,--,float,"Correction factor for capacity factor time series."
 excluder_resolution,m,float,"Resolution on which to perform geographical elibility analysis."
--- a/doc/configtables/opts.csv
+++ b/doc/configtables/opts.csv
@ -1,13 +1,13 @@
-Trigger, Description, Definition, Status
-``nH``; i.e. ``2H``-``6H``, Resample the time-resolution by averaging over every ``n`` snapshots, ``prepare_network``: `average_every_nhours() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L110>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L146>`__), In active use
-``nSEG``; e.g. ``4380SEG``, "Apply time series segmentation with `tsam <https://tsam.readthedocs.io/en/latest/index.html>`_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load.", ``prepare_network``: apply_time_segmentation(), In active use
-``Co2L``, Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``. If a float is appended an overall emission limit relative to the emission level given in ``electricity: co2base`` is added (e.g. ``Co2L0.05`` limits emissisions to 5% of what is given in ``electricity: co2base``), ``prepare_network``: `add_co2limit() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L19>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L154>`__, In active use
-``Ep``, Add cost for a carbon-dioxide price configured in ``costs: emission_prices: co2`` to ``marginal_cost`` of generators (other emission types listed in ``network.carriers`` possible as well), ``prepare_network``: `add_emission_prices() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L24>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L158>`__, In active use
-``Ept``, Add monthly cost for a carbon-dioxide price based on historical values built by the rule ``build_monthly_prices``, In active use
-``CCL``, Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``., ``solve_network``, In active use
-``EQ``, "Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption.", ``solve_network``, In active use
-``ATK``, "Require each node to be autarkic. Example: ``ATK`` removes all lines and links. ``ATKc`` removes all cross-border lines and links.", ``prepare_network``, In active use
-``BAU``, Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities``, ``solve_network``: `add_opts_constraints() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/solve_network.py#L66>`__, Untested
-``SAFE``, Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network., ``solve_network`` `add_opts_constraints() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/solve_network.py#L73>`__, Untested
-``carrier+{c|p|m}factor``,"Alter the capital cost (``c``), installable potential (``p``) or marginal costs (``m``) of a carrier by a factor. Example: ``solar+c0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use
-``CH4L``,"Add an overall absolute gas limit. If configured in ``electricity: gaslimit`` it is given in MWh thermal, if a float is appended, the overall gaslimit is assumed to be given in TWh thermal (e.g. ``CH4L200`` limits gas dispatch to 200 TWh termal)", ``prepare_network``: ``add_gaslimit()``, In active use
+Trigger, Description, Definition, Status
+``nH``; i.e. ``2H``-``6H``, Resample the time-resolution by averaging over every ``n`` snapshots, ``prepare_network``: `average_every_nhours() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L110>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L146>`__), In active use
+``nSEG``; e.g. ``4380SEG``,"Apply time series segmentation with `tsam <https://tsam.readthedocs.io/en/latest/index.html>`_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load.", ``prepare_network``: apply_time_segmentation(), In active use
+``Co2L``,Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``. If a float is appended an overall emission limit relative to the emission level given in ``electricity: co2base`` is added (e.g. ``Co2L0.05`` limits emissisions to 5% of what is given in ``electricity: co2base``), ``prepare_network``: `add_co2limit() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L19>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L154>`__, In active use
+``Ep``,Add cost for a carbon-dioxide price configured in ``costs: emission_prices: co2`` to ``marginal_cost`` of generators (other emission types listed in ``network.carriers`` possible as well), ``prepare_network``: `add_emission_prices() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L24>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L158>`__, In active use
+``Ept``,Add monthly cost for a carbon-dioxide price based on historical values built by the rule ``build_monthly_prices``, In active use,
+``CCL``,Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``., ``solve_network``, In active use
+``EQ``,Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption., ``solve_network``, In active use
+``ATK``,Require each node to be autarkic. Example: ``ATK`` removes all lines and links. ``ATKc`` removes all cross-border lines and links., ``prepare_network``, In active use
+``BAU``,Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities``, ``solve_network``: `add_opts_constraints() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/solve_network.py#L66>`__, Untested
+``SAFE``,Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network., ``solve_network`` `add_opts_constraints() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/solve_network.py#L73>`__, Untested
+``carrier+{c|p|m}factor``,"Alter the capital cost (``c``), installable potential (``p``) or marginal costs (``m``) of a carrier by a factor. Example: ``solar+c0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use
+``CH4L``,"Add an overall absolute gas limit. If configured in ``electricity: gaslimit`` it is given in MWh thermal, if a float is appended, the overall gaslimit is assumed to be given in TWh thermal (e.g. ``CH4L200`` limits gas dispatch to 200 TWh termal)", ``prepare_network``: ``add_gaslimit()``, In active use
--- a/doc/configtables/plotting.csv
+++ b/doc/configtables/plotting.csv
@ -1,6 +1,9 @@
 ,Unit,Values,Description
 map,,,
 -- boundaries,°,"[x1,x2,y1,y2]",Boundaries of the map plots in degrees latitude (y) and longitude (x)
+projection,,,,
+-- name,--,"Valid Cartopy projection name","See https://scitools.org.uk/cartopy/docs/latest/reference/projections.html for list of available projections."
+-- args,--,--,"Other entries under 'projection' are passed as keyword arguments to the projection constructor, e.g. ``central_longitude: 10.``."
 costs_max,bn Euro,float,Upper y-axis limit in cost bar plots.
 costs_threshold,bn Euro,float,Threshold below which technologies will not be shown in cost bar plots.
 energy_max,TWh,float,Upper y-axis limit in energy bar plots.
--- a/doc/configtables/run.csv
+++ b/doc/configtables/run.csv
@ -1,5 +1,8 @@
 ,Unit,Values,Description
-name,--,"any string","Specify a name for your run. Results will be stored under this name."
-disable_progrssbar,bool,"{true, false}","Switch to select whether progressbar should be disabled."
-shared_resources,bool,"{true, false}","Switch to select whether resources should be shared across runs."
+name,--,str/list,"Specify a name for your run. Results will be stored under this name. If ``scenario: enable:`` is set to ``true``, the name must contain a subset of scenario names defined in ``scenario: file:``. If the name is 'all', all defined scenarios will be run."
+scenarios,,,
+-- enable,bool,"{true, false}","Switch to select whether workflow should generate scenarios based on ``file``."
+-- file,str,,"Path to the scenario yaml file. The scenario file contains config overrides for each scenario. In order to be taken account, ``run: scenarios`` has to be set to ``true`` and ``run: name`` has to be a subset of top level keys given in the scenario file. In order to automatically create a `scenario.yaml` file based on a combination of settings, alter and use the ``config/create_scenarios.py`` script in the ``config`` directory."
+disable_progressbar,bool,"{true, false}","Switch to select whether progressbar should be disabled."
+shared_resources,bool/str,,"Switch to select whether resources should be shared across runs. If a string is passed, this is used as a subdirectory name for shared resources. If set to 'base', only resources before creating the elec.nc file are shared."
 shared_cutouts,bool,"{true, false}","Switch to select whether cutouts should be shared across runs."
--- a/doc/configtables/sector-opts.csv
+++ b/doc/configtables/sector-opts.csv
@ -7,5 +7,5 @@ Trigger, Description, Definition, Status
 ``B``,Add biomass,,In active use
 ``I``,Add industry sector,,In active use
 ``A``,Add agriculture sector,,In active use
-``dist``+``n``,Add distribution grid with investment costs of ``n`` times costs in ``data/costs_{cost_year}.csv``,,In active use
+``dist``+``n``,Add distribution grid with investment costs of ``n`` times costs in ``resources/costs_{cost_year}.csv``,,In active use
 ``seq``+``n``,Sets the CO2 sequestration potential to ``n`` Mt CO2 per year,,In active use
--- a/doc/configtables/sector.csv
+++ b/doc/configtables/sector.csv
@ -1,4 +1,9 @@
 ,Unit,Values,Description
+transport,--,"{true, false}",Flag to include transport sector.
+heating,--,"{true, false}",Flag to include heating sector.
+biomass,--,"{true, false}",Flag to include biomass sector.
+industry,--,"{true, false}",Flag to include industry sector.
+agriculture,--,"{true, false}",Flag to include agriculture sector.
 district_heating,--,,`prepare_sector_network.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/prepare_sector_network.py>`_
 -- potential,--,float,maximum fraction of urban demand which can be supplied by district heating
 -- progress,--,Dictionary with planning horizons as keys., Increase of today's district heating demand to potential maximum district heating share. Progress = 0 means today's district heating share. Progress = 1 means maximum fraction of urban demand is supplied by district heating
@ -62,16 +67,17 @@ tes,--,"{true, false}",Add option for storing thermal energy in large water pits
 tes_tau,,,The time constant used to calculate the decay of thermal energy in thermal energy storage (TES): 1- :math:`e^{-1/24τ}`.
 -- decentral,days,float,The time constant in decentralized thermal energy storage (TES)
 -- central,days,float,The time constant in centralized thermal energy storage (TES)
-boilers,--,"{true, false}",Add option for transforming electricity into heat using resistive heater
+boilers,--,"{true, false}",Add option for transforming gas into heat using gas boilers
+resistive_heaters,--,"{true, false}",Add option for transforming electricity into heat using resistive heaters (independently from gas boilers)
 oil_boilers,--,"{true, false}",Add option for transforming oil into heat using boilers
 biomass_boiler,--,"{true, false}",Add option for transforming biomass into heat using boilers
+overdimension_individual_heating,--,"float",Add option for overdimensioning individual heating systems by a certain factor. This allows them to cover heat demand peaks e.g. 10% higher than those in the data with a setting of 1.1.
 chp,--,"{true, false}",Add option for using Combined Heat and Power (CHP)
 micro_chp,--,"{true, false}",Add option for using Combined Heat and Power (CHP) for decentral areas.
 solar_thermal,--,"{true, false}",Add option for using solar thermal to generate heat.
 solar_cf_correction,--,float,The correction factor for the value provided by the solar thermal profile calculations
 marginal_cost_storage,currency/MWh ,float,The marginal cost of discharging batteries in distributed grids
 methanation,--,"{true, false}",Add option for transforming hydrogen and CO2 into methane using methanation.
-helmeth,--,"{true, false}",Add option for transforming power into gas using HELMETH (Integrated High-Temperature ELectrolysis and METHanation for Effective Power to Gas Conversion)
 coal_cc,--,"{true, false}",Add option for coal CHPs with carbon capture
 dac,--,"{true, false}",Add option for Direct Air Capture (DAC)
 co2_vent,--,"{true, false}",Add option for vent out CO2 from storages to the atmosphere.
@ -80,6 +86,8 @@ hydrogen_fuel_cell,--,"{true, false}",Add option to include hydrogen fuel cell f
 hydrogen_turbine,--,"{true, false}",Add option to include hydrogen turbine for re-electrification. Assuming OCGT technology costs
 SMR,--,"{true, false}",Add option for transforming natural gas into hydrogen and CO2 using Steam Methane Reforming (SMR)
 SMR CC,--,"{true, false}",Add option for transforming natural gas into hydrogen and CO2 using Steam Methane Reforming (SMR) and Carbon Capture (CC)
+regional_methanol_demand,--,"{true, false}",Spatially resolve methanol demand. Set to true if regional CO2 constraints needed.
+regional_oil_demand,--,"{true, false}",Spatially resolve oil demand. Set to true if regional CO2 constraints needed.
 regional_co2 _sequestration_potential,,,
 -- enable,--,"{true, false}",Add option for regionally-resolved geological carbon dioxide sequestration potentials based on `CO2StoP <https://setis.ec.europa.eu/european-co2-storage-database_en>`_.
 -- attribute,--,string,Name of the attribute for the sequestration potential
@ -89,9 +97,11 @@ regional_co2 _sequestration_potential,,,
 -- years_of_storage,years,float,The years until potential exhausted at optimised annual rate
 co2_sequestration_potential,MtCO2/a,float,The potential of sequestering CO2 in Europe per year
 co2_sequestration_cost,currency/tCO2,float,The cost of sequestering a ton of CO2
+co2_sequestration_lifetime,years,int,The lifetime of a CO2 sequestration site
 co2_spatial,--,"{true, false}","Add option to spatially resolve carrier representing stored carbon dioxide. This allows for more detailed modelling of CCUTS, e.g. regarding the capturing of industrial process emissions, usage as feedstock for electrofuels, transport of carbon dioxide, and geological sequestration sites."
 ,,,
 co2network,--,"{true, false}",Add option for planning a new carbon dioxide transmission network
+co2_network_cost_factor,p.u.,float,The cost factor for the capital cost of the carbon dioxide transmission network
 ,,,
 cc_fraction,--,float,The default fraction of CO2 captured with post-combustion capture
 hydrogen_underground _storage,--,"{true, false}",Add options for storing hydrogen underground. Storage potential depends regionally.
@ -104,10 +114,16 @@ min_part_load _methanolisation,per unit of p_nom ,float,The minimum unit dispatc
 use_fischer_tropsch _waste_heat,--,"{true, false}",Add option for using waste heat of Fischer Tropsch in district heating networks
 use_fuel_cell_waste_heat,--,"{true, false}",Add option for using waste heat of fuel cells in district heating networks
 use_electrolysis_waste _heat,--,"{true, false}",Add option for using waste heat of electrolysis in district heating networks
+electricity_transmission _grid,--,"{true, false}",Switch for enabling/disabling the electricity transmission grid.
 electricity_distribution _grid,--,"{true, false}",Add a simplified representation of the exchange capacity between transmission and distribution grid level through a link.
 electricity_distribution _grid_cost_factor,,,Multiplies the investment cost of the electricity distribution grid
 ,,,
 electricity_grid _connection,--,"{true, false}",Add the cost of electricity grid connection for onshore wind and solar
+transmission_efficiency,,,Section to specify transmission losses or compression energy demands of bidirectional links. Splits them into two capacity-linked unidirectional links.
+-- {carrier},--,str,The carrier of the link.
+-- -- efficiency_static,p.u.,float,Length-independent transmission efficiency.
+-- -- efficiency_per_1000km,p.u. per 1000 km,float,Length-dependent transmission efficiency ($\eta^{\text{length}}$)
+-- -- compression_per_1000km,p.u. per 1000 km,float,Length-dependent electricity demand for compression ($\eta \cdot \text{length}$) implemented as multi-link to local electricity bus.
 H2_network,--,"{true, false}",Add option for new hydrogen pipelines
 gas_network,--,"{true, false}","Add existing natural gas infrastructure, incl. LNG terminals, production and entry-points. The existing gas network is added with a lossless transport model. A length-weighted `k-edge augmentation algorithm   <https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation>`_   can be run to add new candidate gas pipelines such that all regions of the model can be connected to the gas network. When activated, all the gas demands are regionally disaggregated as well."
 H2_retrofit,--,"{true, false}",Add option for retrofiting existing pipelines to transport hydrogen.
@ -118,6 +134,14 @@ gas_distribution_grid _cost_factor,,,Multiplier for the investment cost of the g
 ,,,
 biomass_spatial,--,"{true, false}",Add option for resolving biomass demand regionally
 biomass_transport,--,"{true, false}",Add option for transporting solid biomass between nodes
+biogas_upgrading_cc,--,"{true, false}",Add option to capture CO2 from biomass upgrading
 conventional_generation,,,Add a more detailed description of conventional carriers. Any power generation requires the consumption of fuel from nodes representing that fuel.
 biomass_to_liquid,--,"{true, false}",Add option for transforming solid biomass into liquid fuel with the same properties as oil
 biosng,--,"{true, false}",Add option for transforming solid biomass into synthesis gas with the same properties as natural gas
+limit_max_growth,,,
+-- enable,--,"{true, false}",Add option to limit the maximum growth of a carrier
+-- factor,p.u.,float,The maximum growth factor of a carrier (e.g. 1.3 allows  30% larger than max historic growth)
+-- max_growth,,,
+-- -- {carrier},GW,float,The historic maximum growth of a carrier
+-- max_relative_growth,
+-- -- {carrier},p.u.,float,The historic maximum relative growth of a carrier
--- a/doc/configtables/snapshots.csv
+++ b/doc/configtables/snapshots.csv
@ -1,4 +1,4 @@
-,Unit,Values,Description
-start,--,"str or datetime-like; e.g. YYYY-MM-DD","Left bound of date range"
-end,--,"str or datetime-like; e.g. YYYY-MM-DD","Right bound of date range"
-inclusive,--,"One of {'neither', 'both', ‘left’, ‘right’}","Make the time interval closed to the ``left``, ``right``, or both sides ``both`` or neither side ``None``."
+,Unit,Values,Description
+start,--,str or datetime-like; e.g. YYYY-MM-DD,Left bound of date range
+end,--,str or datetime-like; e.g. YYYY-MM-DD,Right bound of date range
+inclusive,--,"One of {'neither', 'both', ‘left’, ‘right’}","Make the time interval closed to the ``left``, ``right``, or both sides ``both`` or neither side ``None``."
--- a/doc/configtables/solar.csv
+++ b/doc/configtables/solar.csv
@ -2,14 +2,14 @@
 cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module can be ERA5 or SARAH-2.","Specifies the directory where the relevant weather data ist stored that is specified at ``atlite/cutouts`` configuration. Both ``sarah`` and ``era5`` work."
 resource,,,
 -- method,--,"Must be 'pv'","A superordinate technology type."
-- panel,--,"One of {'Csi', 'CdTe', 'KANENA'} as defined in `atlite <https://github.com/PyPSA/atlite/tree/master/atlite/resources/solarpanel>`__","Specifies the solar panel technology and its characteristic attributes."
+-- panel,--,"One of {'Csi', 'CdTe', 'KANENA'} as defined in `atlite <https://github.com/PyPSA/atlite/tree/master/atlite/resources/solarpanel>`_ . Can be a string or a dictionary with years as keys which denote the year another turbine model becomes available.","Specifies the solar panel technology and its characteristic attributes."
 -- orientation,,,
 -- -- slope,°,"Realistically any angle in [0., 90.]","Specifies the tilt angle (or slope) of the solar panel. A slope of zero corresponds to the face of the panel aiming directly overhead. A positive tilt angle steers the panel towards the equator."
 -- -- azimuth,°,"Any angle in [0., 360.]","Specifies the `azimuth <https://en.wikipedia.org/wiki/Azimuth>`_ orientation of the solar panel. South corresponds to 180.°."
 capacity_per_sqkm,:math:`MW/km^2`,float,"Allowable density of solar panel placement."
 correction_factor,--,float,"A correction factor for the capacity factor (availability) time series."
 corine,--,"Any subset of the `CORINE Land Cover code list <http://www.eea.europa.eu/data-and-maps/data/corine-land-cover-2006-raster-1/corine-land-cover-classes-and/clc_legend.csv/at_download/file>`_","Specifies areas according to CORINE Land Cover codes which are generally eligible for solar panel placement."
+luisa,--,"Any subset of the `LUISA Base Map codes in Annex 1 <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_","Specifies areas according to the LUISA Base Map codes which are generally eligible for solar panel placement."
 natura,bool,"{true, false}","Switch to exclude `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas. Area is excluded if ``true``."
-potential,--,"One of {'simple', 'conservative'}","Method to compute the maximal installable potential for a node; confer :ref:`renewableprofiles`"
 clip_p_max_pu,p.u.,float,"To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero."
 excluder_resolution,m,float,"Resolution on which to perform geographical elibility analysis."
--- a/doc/configtables/solving.csv
+++ b/doc/configtables/solving.csv
@ -6,12 +6,19 @@ options,,,
 -- skip_iterations,bool,"{'true','false'}","Skip iterating, do not update impedances of branches. Defaults to true."
 -- rolling_horizon,bool,"{'true','false'}","Whether to optimize the network in a rolling horizon manner, where the snapshot range is split into slices of size `horizon` which are solved consecutively."
 -- seed,--,int,Random seed for increased deterministic behaviour.
+-- custom_extra_functionality,--,str,Path to a Python file with custom extra functionality code to be injected into the solving rules of the workflow relative to ``rules`` directory.
+-- io_api,string,"{'lp','mps','direct'}",Passed to linopy and determines the API used to communicate with the solver. With the ``'lp'`` and ``'mps'`` options linopy passes a file to the solver; with the ``'direct'`` option (only supported for HIGHS and Gurobi) linopy uses an in-memory python API resulting in better performance.
 -- track_iterations,bool,"{'true','false'}",Flag whether to store the intermediate branch capacities and objective function values are recorded for each iteration in ``network.lines['s_nom_opt_X']`` (where ``X`` labels the iteration)
 -- min_iterations,--,int,Minimum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run.
 -- max_iterations,--,int,Maximum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run.
 -- transmission_losses,int,[0-9],"Add piecewise linear approximation of transmission losses based on n tangents. Defaults to 0, which means losses are ignored."
 -- linearized_unit_commitment,bool,"{'true','false'}",Whether to optimise using the linearized unit commitment formulation.
 -- horizon,--,int,Number of snapshots to consider in each iteration. Defaults to 100.
+constraints ,,,
+-- CCL,bool,"{'true','false'}",Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``.
+-- EQ,bool/string,"{'false',`n(c| )``; i.e. ``0.5``-``0.7c``}",Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption.
+-- BAU,bool,"{'true','false'}",Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities``
+-- SAFE,bool,"{'true','false'}",Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network.
 solver,,,
 -- name,--,"One of {'gurobi', 'cplex', 'cbc', 'glpk', 'ipopt'}; potentially more possible",Solver to use for optimisation problems in the workflow; e.g. clustering and linear optimal power flow.
 -- options,--,Key listed under ``solver_options``.,Link to specific parameter settings.
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

@ -9,7 +9,7 @@
 Configuration
 ##########################################

-PyPSA-Eur has several configuration options which are documented in this section and are collected in a ``config/config.yaml`` file located in the root directory. Users should copy the provided default configuration (``config/config.default.yaml``) and amend their own modifications and assumptions in the user-specific configuration file (``config/config.yaml``); confer installation instructions at :ref:`defaultconfig`.
+PyPSA-Eur has several configuration options which are documented in this section and are collected in a ``config/config.yaml`` file. This file defines deviations from the default configuration (``config/config.default.yaml``); confer installation instructions at :ref:`defaultconfig`.

 .. _toplevel_cf:

@ -383,7 +383,7 @@ overwrite the existing values.

 .. literalinclude:: ../config/config.default.yaml
   :language: yaml
-   :start-after:   type:
+   :start-after: # docs-load
   :end-before: # docs

 .. csv-table::
@ -561,6 +561,21 @@ The list of available biomass is given by the category in `ENSPRESO_BIOMASS <htt
   use ``min`` in ``p_nom_max:`` for more `
   conservative assumptions.

+.. _adjustments_cf:
+
+``adjustments``
+=============
+
+.. literalinclude:: ../config/config.default.yaml
+   :language: yaml
+   :start-at: adjustments:
+   :end-before: # docs
+
+.. csv-table::
+   :header-rows: 1
+   :widths: 22,7,22,33
+   :file: configtables/adjustments.csv
+
 .. _solving_cf:

 ``solving``
--- a/doc/contributing.rst
+++ b/doc/contributing.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

--- a/doc/costs.rst
+++ b/doc/costs.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

@ -9,7 +9,7 @@ Techno-Economic Assumptions

 The database of cost assumptions is retrieved from the repository
 `PyPSA/technology-data <https://github.com/pypsa/technology-data>`_ and then
-saved to a file ``data/costs_{year}.csv``. The ``config/config.yaml`` provides options
+saved to a file ``resources/costs_{year}.csv``. The ``config/config.yaml`` provides options
 to choose a reference year and use a specific version of the repository.

 .. literalinclude:: ../config/config.default.yaml
@ -50,7 +50,7 @@ Modifying Assumptions
 Some cost assumptions (e.g. marginal cost and capital cost) can be directly
 set in the ``config/config.yaml`` (cf. Section  :ref:`costs_cf`  in
 :ref:`config`). To change cost assumptions in more detail, make a copy of
-``data/costs_{year}.csv`` and reference the new cost file in the ``Snakefile``:
+``resources/costs_{year}.csv`` and reference the new cost file in the ``Snakefile``:

 .. literalinclude:: ../Snakefile
   :start-at: COSTS
--- a/doc/foresight.rst
+++ b/doc/foresight.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2021-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2021-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

@ -41,7 +41,7 @@ Perfect foresight scenarios

 .. warning::

-  Perfect foresight is currently implemented as a first test version.
+  Perfect foresight is currently implemented as an experimental test version.

 For running perfect foresight scenarios, you can adjust the
 ``config/config.perfect.yaml``:
--- a/doc/img/base.png
+++ b/doc/img/base.png
--- a/doc/img/elec_s_X.png
+++ b/doc/img/elec_s_X.png
--- a/doc/img/intro-workflow.png
+++ b/doc/img/intro-workflow.png
--- a/doc/index.rst
+++ b/doc/index.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

@ -35,6 +35,8 @@ PyPSA-Eur: A Sector-Coupled Open Optimisation Model of the European Energy Syste
   :target: https://stackoverflow.com/questions/tagged/pypsa
   :alt: Stackoverflow

+|
+
 PyPSA-Eur is an open model dataset of the European energy system at the
 transmission network level that covers the full ENTSO-E area. It covers demand
 and supply for all energy sectors. From version v0.8.0, PyPSA-Eur includes all
@ -116,7 +118,7 @@ of the individual parts.
    topics we are working on. Please feel free to help or make suggestions.

 This project is currently maintained by the `Department of Digital
-Transformation in Energy Systems <https:/www.ensys.tu-berlin.de>`_ at the
+Transformation in Energy Systems <https://www.tu.berlin/en/ensys>`_ at the
 `Technische Universität Berlin <https://www.tu.berlin>`_. Previous versions were
 developed within the `IAI <http://www.iai.kit.edu>`_ at the `Karlsruhe Institute
 of Technology (KIT) <http://www.kit.edu/english/index.php>`_ which was funded by
@ -209,24 +211,6 @@ If you want to cite a specific PyPSA-Eur version, each release of PyPSA-Eur is s
   :target: https://doi.org/10.5281/zenodo.3520874


-Pre-Built Networks as a Dataset
-===============================
-
-There are pre-built networks available as a dataset on Zenodo as well for every release of PyPSA-Eur.
-
-.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3601881.svg
-   :target: https://doi.org/10.5281/zenodo.3601881
-
-The included ``.nc`` files are PyPSA network files which can be imported with PyPSA via:
-
-.. code:: python
-
-    import pypsa
-
-    filename = "elec_s_1024_ec.nc"  # example
-    n = pypsa.Network(filename)
-
-
 Operating Systems
 =================

--- a/doc/installation.rst
+++ b/doc/installation.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

@ -31,7 +31,7 @@ Install Python Dependencies

 PyPSA-Eur relies on a set of other Python packages to function.
 We recommend using the package manager `mamba <https://mamba.readthedocs.io/en/latest/>`_ to install them and manage your environments.
-For instructions for your operating system follow the ``mamba`` `installation guide <https://mamba.readthedocs.io/en/latest/installation.html>`_.
+For instructions for your operating system follow the ``mamba`` `installation guide <https://mamba.readthedocs.io/en/latest/installation/mamba-installation.html>`_.
 You can also use ``conda`` equivalently.

 The package requirements are curated in the `envs/environment.yaml <https://github.com/PyPSA/pypsa-eur/blob/master/envs/environment.yaml>`_ file.
@ -79,31 +79,9 @@ Nevertheless, you can still use open-source solvers for smaller problems.
    `Instructions how to install a solver in the documentation of PyPSA <https://pypsa.readthedocs.io/en/latest/installation.html#getting-a-solver-for-linear-optimisation>`_

 .. note::
-    The rules :mod:`cluster_network` and :mod:`simplify_network` solve a quadratic optimisation problem for clustering.
-    The open-source solvers Cbc and GlPK cannot handle this. A fallback to Ipopt is implemented in this case, but requires
-    it to be installed. For an open-source solver setup install in your ``conda`` environment on OSX/Linux
-
-    .. code:: bash
-
-        mamba activate pypsa-eur
-        mamba install -c conda-forge ipopt coincbc
-
-    and on Windows
-
-    .. code:: bash
-
-        mamba activate pypsa-eur
-        mamba install -c conda-forge ipopt glpk
-
-    For HiGHS, run
-
-    .. code:: bash
-
-        mamba activate pypsa-eur
-        mamba install -c conda-forge ipopt
-        pip install highspy
-
-    For Gurobi, run
+    The rules :mod:`cluster_network` and :mod:`simplify_network` solve a mixed-integer quadratic optimisation problem for clustering.
+    The open-source solvers HiGHS, Cbc and GlPK cannot handle this. A fallback to SCIP is implemented in this case.
+    For an open-source solver setup install in your ``conda`` environment on OSX/Linux. To install the default solver Gurobi, run

    .. code:: bash

@ -118,11 +96,10 @@ Nevertheless, you can still use open-source solvers for smaller problems.
 Handling Configuration Files
 ============================

-PyPSA-Eur has several configuration options that must be specified in a
-``config/config.yaml`` file located in the root directory. An example configuration
-``config/config.default.yaml`` is maintained in the repository, which will be used to
-automatically create your customisable ``config/config.yaml`` on first use. More
-details on the configuration options are in :ref:`config`.
+PyPSA-Eur has several configuration options that users can specify in a
+``config/config.yaml`` file. The default configuration
+``config/config.default.yaml`` is maintained in the repository. More details on
+the configuration options are in :ref:`config`.

 You can also use ``snakemake`` to specify another file, e.g.
 ``config/config.mymodifications.yaml``, to update the settings of the ``config/config.yaml``.
@ -130,8 +107,3 @@ You can also use ``snakemake`` to specify another file, e.g.
 .. code:: bash

    .../pypsa-eur % snakemake -call --configfile config/config.mymodifications.yaml
-
-.. warning::
-    Users are advised to regularly check their own ``config/config.yaml`` against changes
-    in the ``config/config.default.yaml`` when pulling a new version from the remote
-    repository.
--- a/doc/introduction.rst
+++ b/doc/introduction.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

@ -74,7 +74,7 @@ what data to retrieve and what files to produce. Details are explained in
 :ref:`wildcards` and :ref:`scenario`.

 The model also has several further configuration options collected in the
-``config/config.yaml`` file located in the root directory, which that are not part of
+``config/config.default.yaml`` file located in the root directory, which that are not part of
 the scenarios. Options are explained in :ref:`config`.

 Folder Structure
@ -89,8 +89,8 @@ Folder Structure
 - ``results``: Stores the solved PyPSA network data, summary files and plots.
 - ``logs``: Stores log files.
 - ``benchmarks``: Stores ``snakemake`` benchmarks.
- ``test``: Includes the test configuration files used for continuous integration.
 - ``doc``: Includes the documentation of PyPSA-Eur.
+- ``graphics``: Includes some graphics for the documentation of PyPSA-Eur.

 System Requirements
 ===================
--- a/doc/licenses.rst
+++ b/doc/licenses.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2023-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

--- a/doc/limitations.rst
+++ b/doc/limitations.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

--- a/doc/make.bat
+++ b/doc/make.bat
@ -1,4 +1,4 @@
-REM SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+REM SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors
 REM SPDX-License-Identifier: MIT

@ECHO OFF
--- a/doc/plotting.rst
+++ b/doc/plotting.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

@ -22,7 +22,22 @@ Rule ``plot_summary``

 .. _map_plot:

-Rule ``plot_network``
-========================
+Rule ``plot_power_network``
+===========================

-.. automodule:: plot_network
+.. automodule:: plot_power_network
+
+Rule ``plot_power_network_perfect``
+===================================
+
+.. automodule:: plot_power_network_perfect
+
+Rule ``plot_hydrogen_network``
+==============================
+
+.. automodule:: plot_hydrogen_network
+
+Rule ``plot_gas_network``
+=========================
+
+.. automodule:: plot_gas_network
--- a/doc/preparation.rst
+++ b/doc/preparation.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

@ -94,6 +94,13 @@ Rule ``build_electricity_demand``

 .. automodule:: build_electricity_demand

+.. _monthlyprices:
+
+Rule ``build_monthly_prices``
+=============================
+
+.. automodule:: build_monthly_prices
+
 .. _ship:

 Rule ``build_ship_raster``
@ -102,6 +109,12 @@ Rule ``build_ship_raster``

 .. automodule:: build_ship_raster

+.. _availabilitymatrixmdua:
+
+Rule ``determine_availability_matrix_MD_UA``
+============================================
+
+.. automodule:: determine_availability_matrix_MD_UA

 .. _renewableprofiles:

--- a/doc/publications.bib
+++ b/doc/publications.bib
@ -1,5 +1,5 @@
@Comment{
-SPDX-FileCopyrightText: 2023 The PyPSA-Eur Authors
+SPDX-FileCopyrightText: 2023-2024 The PyPSA-Eur Authors

 SPDX-License-Identifier: CC0-1.0
 }
--- a/doc/publications.rst
+++ b/doc/publications.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2023-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

@ -10,37 +10,596 @@ Release Notes
 Upcoming Release
 ================

-* Updated Global Energy Monitor LNG terminal data to March 2023 version.
+* Upgrade to Snakemake v8.5+. This version is the new minimum version required.
+  To upgrade an existing environment, run ``conda install -c bioconda
+  snakemake-minimal">=8.5"`` and ``pip install snakemake-storage-plugin-http``
+  (https://github.com/PyPSA/pypsa-eur/pull/825).

-* For industry distribution, use EPRTR as fallback if ETS data is not available.
+* Corrected a bug leading to power plants operating after their DateOut
+  (https://github.com/PyPSA/pypsa-eur/pull/958). Added additional grouping years
+  before 1980.

-* The minimum capacity for renewable generators when using the myopic option has been fixed.
+* The Eurostat data was updated to the 2023 version in :mod:`build_energy_totals`.

-* Files downloaded from zenodo are now write-protected to prevent accidental re-download.
+* The latest `Swiss energy totals
+  <https://www.bfe.admin.ch/bfe/de/home/versorgung/statistik-und-geodaten/energiestatistiken/energieverbrauch-nach-verwendungszweck.html/>`_
+  have been updated to the 2023 version.

-* Files extracted from sector-coupled data bundle have been moved from ``data/`` to ``data/sector-bundle``.
+* The JRC-IDEES data is only available until 2015. For energy totals years (``energy: energy_totals_year``) after
+  2015, the data scaled using the ratio of Eurostat data reported for the energy
+  totals year and 2015.

-* New feature multi-decade optimisation with  perfect foresight.
+* The default energy totals year (``energy: energy_totals_year``) was updated to 2019.

-* It is now possible to specify years for biomass potentials which do not exist
-  in the JRC-ENSPRESO database, e.g. 2037. These are linearly interpolated.
+* Upgrade default techno-economic assumptions to ``technology-data`` v0.8.1.

-* In pathway mode, the biomass potential is linked to the investment year.
+* Linearly interpolate missing investment periods in year-dependent
+  configuration options.

-* Rule ``purge`` now initiates a dialog to confirm if purge is desired.
+* Added new scenario management that supports the simultaneous execution of
+  multiple scenarios with a single ``snakemake`` call. For this purpose, a
+  ``scenarios.yaml`` file is introduced which contains customizable scenario
+  names with configuration overrides. To enable it, set the ``run: scenarios:
+  true`` and define the list of scenario names to run under ``run: name:`` in
+  the configuration file. The latter must be a subset of toplevel keys in the
+  scenario file.

-* Rule ``retrieve_irena`` get updated values for renewables capacities.
+  - To get started, a scenarios template file ``config/scenarios.template.yaml``
+    is included in the repository, which is copied to ``config/scenarios.yaml``
+    on first use.

-* Split configuration to enable SMR and SMR CC.
+  - The scenario file can be changed via ``run: scenarios: file:``.

-* The ``mock_snakemake`` function can now be used with a Snakefile from a different directory using the new ``root_dir`` argument.
+  - If scenario management is activated with ``run: scenarios: enable: true``, a
+    new wildcard ``{run}`` is introduced. This means that the configuration
+    settings may depend on the new ``{run}`` wildcard. Therefore, a new
+    ``config_provider()`` function is used in the ``Snakefile`` and ``.smk``
+    files, which takes wildcard values into account. The calls to the ``config``
+    object have been reduced in ``.smk`` files since there is no awareness of
+    wildcard values outside rule definitions.

-* Added Enhanced Geothermal Systems for electricity generation.
+  - The scenario files can also be programmatically created using the template
+    script ``config/create_scenarios.py``. This script can be run with
+    ``snakemake -j1 create_scenarios`` and creates the scenarios file referenced
+    under ``run: scenarios: file:``.

+  - The setting ``run: name: all`` will run all scenarios in
+    ``config/scenarios.yaml``. Otherwise, it will run those passed as list in
+    ``run: name:`` as long as ``run: scenarios: enable: true``.
+
+  - The setting ``run: shared_resources:`` indicates via a boolean whether the
+    resources should be encapsulated by the ``run: name:``. The special setting
+    ``run: shared_resources: base`` shares resources until ``add_electricity``
+    that do not contain wildcards other than ``{"technology", "year",
+    "scope"}``.
+
+  - Added new configuration options for all ``{opts}`` and ``{sector_opts}``
+    wildcard values to create a unique configuration file (``config.yaml``) per
+    PyPSA network file. This is done with the help of a new function
+    ``update_config_from_wildcards()`` which parses configuration settings from
+    wildcards and updates the ``snakemake.config`` object. These updated
+    configuration settings are used in the scripts rather than directly parsed
+    values from ``snakemake.wildcards``.
+
+  - The cost data was moved from ``data/costs_{year}.csv`` to
+    ``resources/costs_{year}.csv`` since it depends on configuration settings.
+    The ``retrieve_cost_data`` rule was changed to calling a Python script.
+
+  - Moved time clustering settings to ``clustering: temporal:`` from
+    ``snapshots:`` so that the latter is only used to define the
+    ``pandas.DatetimeIndex`` which simplifies the scenario management.
+
+  - Collection rules get a new wildcard ``run=config["run"]["name"]`` so they
+    can collect outputs across different scenarios.
+
+  - **Warning:** One caveat remains for the scenario management with myopic or
+    perfect foresight pathway optimisation. The first investment period must be
+    shared across all scenarios. The reason is that the ``wildcard_constraints``
+    defined for the rule ``add_existing_baseyear`` do not accept wildcard-aware
+    input functions (cf.
+    `https://github.com/snakemake/snakemake/issues/2703`_).
+
+* The outputs of the rule ``retrieve_gas_infrastructure_data`` no longer
+  marked as ``protected()`` as the download size is small.
+
+* Bugfix: allow modelling sector-coupled landlocked regions. (Fixed handling of offshore wind.)
+
+* Adapt the disabling of transmission expansion in myopic foresight optimisations when limit is already reached to also handle cost limits.
+
+PyPSA-Eur 0.10.0 (19th February 2024)
+=====================================
+
+**New Features**
+
+* Improved representation of industry transition pathways. A new script was
+  added to interpolate industry sector ratios from today's status quo to future
+  systems (i.e. specific emissions and demands for energy and feedstocks). For
+  each country we gradually switch industry processes from today's specific
+  energy carrier usage per ton material output to the best-in-class energy
+  consumption of tomorrow. This is done on a per-country basis. The ratio of
+  today to tomorrow's energy consumption is set with the ``industry:
+  sector_ratios_fraction_future:`` parameter
+  (https://github.com/PyPSA/pypsa-eur/pull/929).
+
+* Add new default to overdimension heating in individual buildings. This allows
+  them to cover heat demand peaks e.g. 10% higher than those in the data. The
+  disadvantage of manipulating the costs is that the capacity is then not quite
+  right. This way at least the costs are right
+  (https://github.com/PyPSA/pypsa-eur/pull/918).
+
+* Allow industrial coal demand to be regional so its emissions can be included
+  in regional emission limits (https://github.com/PyPSA/pypsa-eur/pull/923).
+
+* Add option to specify to set a default heating lifetime for existing heating
+  (``existing_capacities: default_heating_lifetime:``)
+  (https://github.com/PyPSA/pypsa-eur/pull/918).
+
+* Added option to specify turbine and solar panel models for specific years as a
+  dictionary (e.g. ``renewable: onwind: resource: turbine:``). The years will be
+  interpreted as years from when the the corresponding turbine model substitutes
+  the previous model for new installations. This will only have an effect on
+  workflows with foresight ``"myopic"`` and still needs to be added foresight
+  option ``"perfect"`` (https://github.com/PyPSA/pypsa-eur/pull/912).
+
+* New configuration option ``everywhere_powerplants`` to build conventional
+  powerplants everywhere, irrespective of existing powerplants locations, in the
+  network (https://github.com/PyPSA/pypsa-eur/pull/850).
+
+* Add the option to customise map projection in plotting config under
+  ``plotting: projection: name`` (https://github.com/PyPSA/pypsa-eur/pull/898).
+
+* Add support for the linopy ``io_api`` option under ``solving: options:
+  io_api:``. Set to ``"direct"`` to increase model reading and writing
+  performance for the highs and gurobi solvers on slow file systems
+  (https://github.com/PyPSA/pypsa-eur/pull/892).
+
+* It is now possible to determine the directory for shared resources by setting
+  `shared_resources` to a string (https://github.com/PyPSA/pypsa-eur/pull/906).
+
+* Improve ``mock_snakemake()`` for usage in Snakemake modules
+  (https://github.com/PyPSA/pypsa-eur/pull/869).
+
+**Breaking Changes**
+
+* Remove long-deprecated function ``attach_extendable_generators`` in
+  :mod:`add_electricity`.
+
+* Remove option for wave energy as technology data is not maintained.
+
+* The order of buses (bus0, bus1, ...) for DAC components has changed to meet
+  the convention of the other components. Therefore, `bus0` refers to the
+  electricity bus (input), `bus1` to the heat bus (input), 'bus2' to the CO2
+  atmosphere bus (input), and `bus3` to the CO2 storage bus (output)
+  (https://github.com/PyPSA/pypsa-eur/pull/901).
+
+**Changes**
+
+* Upgrade default techno-economic assumptions to ``technology-data`` v0.8.0.
+
+* Update hydrogen pipeline losses to latest data from Danish Energy Agency
+  (https://github.com/PyPSA/pypsa-eur/pull/933).
+
+* Move building of daily heat profile to its own rule
+  :mod:`build_hourly_heat_demand` from :mod:`prepare_sector_network`
+  (https://github.com/PyPSA/pypsa-eur/pull/884).
+
+* In :mod:`build_energy_totals`, district heating shares are now reported in a
+  separate file (https://github.com/PyPSA/pypsa-eur/pull/884).
+
+* Move calculation of district heating share to its own rule
+  :mod:`build_district_heat_share`
+  (https://github.com/PyPSA/pypsa-eur/pull/884).
+
+* Move building of distribution of existing heating to own rule
+  :mod:`build_existing_heating_distribution`. This makes the distribution of
+  existing heating to urban/rural, residential/services and spatially more
+  transparent (https://github.com/PyPSA/pypsa-eur/pull/884).
+
+* Default settings for recycling rates and primary product shares of high-value
+  chemicals have been set in accordance with the values used in `Neumann et al.
+  (2023) <https://doi.org/10.1016/j.joule.2023.06.016>`_ linearly interpolated
+  between 2020 and 2050. The recycling rates are based on data from `Agora
+  Energiewende (2021)
+  <https://static.agora-energiewende.de/fileadmin/Projekte/2021/2021_02_EU_CEAP/A-EW_254_Mobilising-circular-economy_study_WEB.pdf>`_.
+
+* Air-sourced heat pumps can now also be built in rural areas. Previously, only
+  ground-sourced heat pumps were considered for this category
+  (https://github.com/PyPSA/pypsa-eur/pull/890).
+
+* The default configuration ``config/config.default.yaml`` is now automatically
+  used as a base configuration file. The file ``config/config.yaml`` can now be
+  used to only define deviations from the default configuration. The
+  ``config/config.default.yaml`` is still copied into ``config/config.yaml`` on
+  first usage (https://github.com/PyPSA/pypsa-eur/pull/925).
+
+* Regions are assigned to all buses with unique coordinates in the network with
+  a preference given to substations. Previously, only substations had assigned
+  regions, but this could lead to issues when a high spatial resolution was
+  applied (https://github.com/PyPSA/pypsa-eur/pull/922).
+
+* Define global constraint for CO2 emissions on the final state of charge of the
+  CO2 atmosphere store. This gives a more sparse constraint that should improve
+  the performance of the solving process
+  (https://github.com/PyPSA/pypsa-eur/pull/862).
+
+* Switched the energy totals year from 2011 to 2013 to comply with the assumed
+  default weather year (https://github.com/PyPSA/pypsa-eur/pull/934).
+
+* Cluster residential and services heat buses by default. Can be disabled with
+  ``cluster_heat_buses: false`` (https://github.com/PyPSA/pypsa-eur/pull/877).
+
+* The rule ``plot_network`` has been split into separate rules for plotting
+  electricity, hydrogen and gas networks
+  (https://github.com/PyPSA/pypsa-eur/pull/900).
+
+* To determine the optimal topology to meet the number of clusters, the workflow
+  used pyomo in combination with ``ipopt`` or ``gurobi``. This dependency has
+  been replaced by using ``linopy`` in combination with ``scipopt`` or
+  ``gurobi``. The environment file has been updated accordingly
+  (https://github.com/PyPSA/pypsa-eur/pull/903).
+
+* The ``highs`` solver was added to the default environment file.
+
+* New default solver settings for COPT solver
+  (https://github.com/PyPSA/pypsa-eur/pull/882).
+
+* Data retrieval rules now use their own minimal conda environment. This can
+  avoid unnecessary reruns of the workflow
+  (https://github.com/PyPSA/pypsa-eur/pull/888).
+
+* Merged two OPSD time series data versions into such that the option ``load:
+  power_statistics:`` becomes superfluous and was hence removed
+  (https://github.com/PyPSA/pypsa-eur/pull/924).
+
+* The filtering of power plants in the ``config.default.yaml`` has been updated
+  regarding phased-out power plants in 2023.
+
+* Include all countries in ammonia production resource. This is so that the full
+  EU28 ammonia demand can be correctly subtracted in the rule
+  :mod:`build_industry_sector_ratios`
+  (https://github.com/PyPSA/pypsa-eur/pull/931).
+
+* Correctly source the existing heating technologies for buildings since the
+  source URL has changed. It represents the year 2012 and is only for buildings,
+  not district heating (https://github.com/PyPSA/pypsa-eur/pull/918).
+
+* Add warning when BEV availability weekly profile has negative values in
+  `build_transport_demand` (https://github.com/PyPSA/pypsa-eur/pull/858).
+
+* Time series clipping for very small values was added for Links
+  (https://github.com/PyPSA/pypsa-eur/pull/870).
+
+* A ``test.sh`` script was added to the repository to run the tests locally.
+
+* The CI now tests additionally against ``master`` versions of PyPSA, atlite and
+  powerplantmatching (https://github.com/PyPSA/pypsa-eur/pull/904).
+
+* A function ``sanitize_locations()`` was added to improve the coverage of the
+  ``location`` attribute of network components.

 **Bugs and Compatibility**

-* A bug preventing custom powerplants specified in ``data/custom_powerplants.csv`` was fixed. (https://github.com/PyPSA/pypsa-eur/pull/732)
+* Bugfix: Do not reduce district heat share when building population-weighted
+  energy statistics. Previously the district heating share was being multiplied
+  by the population weighting, reducing the DH share with multiple nodes
+  (https://github.com/PyPSA/pypsa-eur/pull/884).
+
+* Bugfix: The industry coal emissions for industry were not properly tracked
+  (https://github.com/PyPSA/pypsa-eur/pull/923).
+
+* Bugfix: Correct units of subtracted chlorine and methanol demand in
+  :mod:`build_industry_sector_ratios`
+  (https://github.com/PyPSA/pypsa-eur/pull/930).
+
+* Various minor bugfixes to the perfect foresight workflow, though perfect
+  foresight must still be considered experimental
+  (https://github.com/PyPSA/pypsa-eur/pull/910).
+
+* Fix plotting of retrofitted hydrogen pipelines with myopic pathway
+  optimisation (https://github.com/PyPSA/pypsa-eur/pull/937).
+
+* Bugfix: Correct technology keys for the electricity production plotting to
+  work out the box.
+
+* Bugfix: Assure entering of code block which corrects Norwegian heat demand
+  (https://github.com/PyPSA/pypsa-eur/pull/870).
+
+* Stacktrace of uncaught exceptions should now be correctly included inside log
+  files (via `configure_logging(..)`)
+  (https://github.com/PyPSA/pypsa-eur/pull/875).
+
+* Bugfix: Correctly read out number of solver threads from configuration file
+  (https://github.com/PyPSA/pypsa-eur/pull/889).
+
+* Made copying default config file compatible with snakemake module
+  (https://github.com/PyPSA/pypsa-eur/pull/894).
+
+* Compatibility with ``pandas=2.2``
+  (https://github.com/PyPSA/pypsa-eur/pull/861).
+
+Special thanks for this release to Koen van Greevenbroek (`@koen-vg
+<https://github.com/koen-vg>`_) for various new features, bugfixes and taking
+care of deprecations.
+
+
+PyPSA-Eur 0.9.0 (5th January 2024)
+==================================
+
+**New Features**
+
+* Add option to specify losses for bidirectional links, e.g. pipelines or HVDC
+  links, in configuration file under ``sector: transmission_efficiency:``. Users
+  can specify static or length-dependent values as well as a length-dependent
+  electricity demand for compression, which is implemented as a multi-link to
+  the local electricity buses. The bidirectional links will then be split into
+  two unidirectional links with linked capacities (https://github.com/PyPSA/pypsa-eur/pull/739).
+
+* Merged option to extend geographical scope to Ukraine and Moldova. These
+  countries are excluded by default and is currently constrained to power-sector
+  only parts of the workflow. A special config file
+  `config/config.entsoe-all.yaml` was added as an example to run the workflow
+  with all ENTSO-E member countries (including observer members like Ukraine and
+  Moldova). Moldova can currently only be included in conjunction with Ukraine
+  due to the absence of demand data. The Crimean power system is manually
+  reconnected to the main Ukrainian grid with the configuration option
+  `reconnect_crimea` (https://github.com/PyPSA/pypsa-eur/pull/321).
+
+* New experimental support for multi-decade optimisation with perfect foresight
+  (``foresight: perfect``). Maximum growth rates for carriers, global carbon
+  budget constraints and emission constraints for particular investment periods.
+
+* Add option to reference an additional source file where users can specify
+  custom ``extra_functionality`` constraints in the configuration file. The
+  default setting points to an empty hull at
+  ``data/custom_extra_functionality.py`` (https://github.com/PyPSA/pypsa-eur/pull/824).
+
+* Add locations, capacities and costs of existing gas storage using Global
+  Energy Monitor's `Europe Gas Tracker
+  <https://globalenergymonitor.org/projects/europe-gas-tracker>`_
+  (https://github.com/PyPSA/pypsa-eur/pull/835).
+
+* Add option to use `LUISA Base Map
+  <https://publications.jrc.ec.europa.eu/repository/handle/JRC124621>`_ 50m land
+  coverage dataset for land eligibility analysis in
+  :mod:`build_renewable_profiles`. Settings are analogous to the CORINE dataset
+  but with the key ``luisa:`` in the configuration file. To leverage the
+  dataset's full advantages, set the excluder resolution to 50m
+  (``excluder_resolution: 50``). For land category codes, see `Annex 1 of the
+  technical documentation
+  <https://publications.jrc.ec.europa.eu/repository/bitstream/JRC124621/technical_report_luisa_basemap_2018_v7_final.pdf>`_
+  (https://github.com/PyPSA/pypsa-eur/pull/842).
+
+* Add option to capture CO2 contained in biogas when upgrading (``sector:
+  biogas_to_gas_cc``) (https://github.com/PyPSA/pypsa-eur/pull/615).
+
+* If load shedding is activated, it is now applied to all carriers, not only
+  electricity (https://github.com/PyPSA/pypsa-eur/pull/784).
+
+* Add option for heat vents in district heating (``sector:
+  central_heat_vent:``). The combination of must-run conditions for some
+  power-to-X processes, waste heat usage enabled and decreasing heating demand,
+  can lead to infeasibilities in pathway optimisation for some investment
+  periods since larger Fischer-Tropsch capacities are needed in early years but
+  the waste heat exceeds the heat demand in later investment periods.
+  (https://github.com/PyPSA/pypsa-eur/pull/791).
+
+* Allow possibility to go from copperplated to regionally resolved methanol and
+  oil demand with switches ``sector: regional_methanol_demand: true`` and
+  ``sector: regional_oil_demand: true``. This allows nodal/regional CO2
+  constraints to be applied (https://github.com/PyPSA/pypsa-eur/pull/827).
+
+* Allow retrofitting of existing gas boilers to hydrogen boilers in pathway
+  optimisation.
+
+* Add option to add time-varying CO2 emission prices (electricity-only, ``costs:
+  emission_prices: co2_monthly_prices: true``). This is linked to the new
+  ``{opts}`` wildcard option ``Ept``.
+
+* Network clustering can now consider efficiency classes when aggregating
+  carriers. The option ``clustering: consider_efficiency_classes:`` aggregates
+  each carriers into the top 10-quantile (high), the bottom 90-quantile (low),
+  and everything in between (medium).
+
+* Added option ``conventional: dynamic_fuel_price:`` to consider the monthly
+  fluctuating fuel prices for conventional generators. Refer to the CSV file
+  ``data/validation/monthly_fuel_price.csv``.
+
+* For hydro-electricity, add switches ``flatten_dispatch`` to consider an upper
+  limit for the hydro dispatch. The limit is given by the average capacity
+  factor plus the buffer given in  ``flatten_dispatch_buffer``.
+
+* Extend options for waste heat usage from Haber-Bosch, methanolisation and
+  methanation (https://github.com/PyPSA/pypsa-eur/pull/834).
+
+* Add new ``sector_opts`` wildcard option "nowasteheat" to disable all waste
+  heat usage (https://github.com/PyPSA/pypsa-eur/pull/834).
+
+* Add new rule ``retrieve_irena`` to automatically retrieve up-to-date values
+  for existing renewables capacities (https://github.com/PyPSA/pypsa-eur/pull/756).
+
+* Print Irreducible Infeasible Subset (IIS) if model is infeasible. Only for
+  solvers with IIS support (https://github.com/PyPSA/pypsa-eur/pull/841).
+
+* More wildcard options now have a corresponding config entry. If the wildcard
+  is given, then its value is used. If the wildcard is not given but the options
+  in config are enabled, then the value from config is used. If neither is
+  given, the options are skipped (https://github.com/PyPSA/pypsa-eur/pull/827).
+
+* Validate downloads from Zenodo using MD5 checksums. This identifies corrupted
+  or incomplete downloads (https://github.com/PyPSA/pypsa-eur/pull/821).
+
+* Add rule ``sync`` to synchronise with a remote machine using the ``rsync``
+  library. Configuration settings are found under ``remote:``.
+
+**Breaking Changes**
+
+* Remove all negative loads on the ``co2 atmosphere`` bus representing emissions
+  for e.g. fixed fossil demands for transport oil. Instead these are handled
+  more transparently with a fixed transport oil demand and a link taking care of
+  the emissions to the ``co2 atmosphere`` bus. This is also a preparation for
+  endogenous transport optimisation, where demand will be subject to
+  optimisation (e.g. fuel switching in the transport sector)
+  (https://github.com/PyPSA/pypsa-eur/pull/827).
+
+* Process emissions from steam crackers (i.e. naphtha processing for HVC) are
+  now piped from the consumption link to the process emissions bus where the
+  model can decide about carbon capture. Previously the process emissions for
+  naphtha were a fixed load (https://github.com/PyPSA/pypsa-eur/pull/827).
+
+* Distinguish between stored and sequestered CO2. Stored CO2 is stored
+  overground in tanks and can be used for CCU (e.g. methanolisation).
+  Sequestered CO2 is stored underground and can no longer be used for CCU. This
+  distinction is made because storage in tanks is more expensive than
+  underground storage. The link that connects stored and sequestered CO2 is
+  unidirectional (https://github.com/PyPSA/pypsa-eur/pull/844).
+
+* Files extracted from sector-coupled data bundle have been moved from ``data/``
+  to ``data/sector-bundle``.
+
+* Split configuration to enable SMR and SMR CC (``sector: smr:`` and ``sector:
+  smr_cc:``) (https://github.com/PyPSA/pypsa-eur/pull/757).
+
+* Add separate option to add resistive heaters to the technology choices
+  (``sector: resistive_heaters:``). Previously they were always added when
+  boilers were added (https://github.com/PyPSA/pypsa-eur/pull/808).
+
+* Remove HELMETH option (``sector: helmeth:``).
+
+* Remove "conservative" renewable potentials estimation option
+  (https://github.com/PyPSA/pypsa-eur/pull/838).
+
+* With this release we stop posting updates to the network pre-builts.
+
+**Changes**
+
+* Updated Global Energy Monitor LNG terminal data to March 2023 version
+  (https://github.com/PyPSA/pypsa-eur/pull/707).
+
+* For industry distribution, use EPRTR as fallback if ETS data is not available
+  (https://github.com/PyPSA/pypsa-eur/pull/721).
+
+* It is now possible to specify years for biomass potentials which do not exist
+  in the JRC-ENSPRESO database, e.g. 2037. These are linearly interpolated
+  (https://github.com/PyPSA/pypsa-eur/pull/744).
+
+* In pathway mode, the biomass potential is linked to the investment year
+  (https://github.com/PyPSA/pypsa-eur/pull/744).
+
+* Increase allowed deployment density of solar to 5.1 MW/sqkm by default.
+
+* Default to full electrification of land transport by 2050.
+
+* Provide exogenous transition settings in 5-year steps.
+
+* Default to approximating transmission losses in HVAC lines
+  (``transmission_losses: 2``).
+
+* Use electrolysis waste heat by default.
+
+* Set minimum part loads for PtX processes to 30% for methanolisation and
+  methanation, and to 70% for Fischer-Tropsch synthesis.
+
+* Add VOM as marginal cost to PtX processes
+  (https://github.com/PyPSA/pypsa-eur/pull/830).
+
+* Add pelletizing costs for biomass boilers (https://github.com/PyPSA/pypsa-eur/pull/833).
+
+* Update default offshore wind turbine model to "NREL Reference 2020 ATB 5.5 MW"
+  (https://github.com/PyPSA/pypsa-eur/pull/832).
+
+* Switch to using hydrogen and electricity inputs for Haber-Bosch from
+  https://github.com/PyPSA/technology-data (https://github.com/PyPSA/pypsa-eur/pull/831).
+
+* The configuration setting for country focus weights when clustering the
+  network has been moved from ``focus_weights:`` to ``clustering:
+  focus_weights:``. Backwards compatibility to old config files is maintained
+  (https://github.com/PyPSA/pypsa-eur/pull/794).
+
+* The ``mock_snakemake`` function can now be used with a Snakefile from a
+  different directory using the new ``root_dir`` argument
+  (https://github.com/PyPSA/pypsa-eur/pull/771).
+
+* Rule ``purge`` now initiates a dialog to confirm if purge is desired
+  (https://github.com/PyPSA/pypsa-eur/pull/745).
+
+* Files downloaded from zenodo are now write-protected to prevent accidental
+  re-download (https://github.com/PyPSA/pypsa-eur/pull/730).
+
+* Performance improvements for rule ``build_ship_raster``
+  (https://github.com/PyPSA/pypsa-eur/pull/845).
+
+* Improve time logging in :mod:`build_renewable_profiles`
+  (https://github.com/PyPSA/pypsa-eur/pull/837).
+
+* In myopic pathway optimisation, disable power grid expansion if line volume
+  already hit (https://github.com/PyPSA/pypsa-eur/pull/840).
+
+* JRC-ENSPRESO data is now downloaded from a Zenodo mirror because the link was
+  unreliable (https://github.com/PyPSA/pypsa-eur/pull/801).
+
+* Add focus weights option for clustering to documentation
+  (https://github.com/PyPSA/pypsa-eur/pull/781).
+
+* Add proxy for biomass transport costs if no explicit biomass transport network
+  is considered (https://github.com/PyPSA/pypsa-eur/pull/711).
+
+**Bugs and Compatibility**
+
+* The minimum PyPSA version is now 0.26.1.
+
+* Update to ``tsam>=0.2.3`` for performance improvents in temporal clustering.
+
+* Pin ``snakemake`` version to below 8.0.0, as the new version is not yet
+  supported. The next release will switch to the requirement ``snakemake>=8``.
+
+* Bugfix: Add coke and coal demand for integrated steelworks
+  (https://github.com/PyPSA/pypsa-eur/pull/718).
+
+* Bugfix: Make :mod:`build_renewable_profiles` consider subsets of cutout time
+  scope (https://github.com/PyPSA/pypsa-eur/pull/709).
+
+* Bugfix: In :mod:`simplify network`, remove 'underground' column to avoid
+  consense error (https://github.com/PyPSA/pypsa-eur/pull/714).
+
+* Bugfix: Fix in :mod:`add_existing_baseyear` to account for the case when there
+  is no rural heating demand for some nodes in network
+  (https://github.com/PyPSA/pypsa-eur/pull/706).
+
+* Bugfix: The unit of the capital cost of Haber-Bosch plants was corrected
+  (https://github.com/PyPSA/pypsa-eur/pull/829).
+
+* The minimum capacity for renewable generators when using the myopic option has
+  been fixed (https://github.com/PyPSA/pypsa-eur/pull/728).
+
+* Compatibility for running with single node and single country
+  (https://github.com/PyPSA/pypsa-eur/pull/839).
+
+* A bug preventing the addition of custom powerplants specified in
+  ``data/custom_powerplants.csv`` was fixed.
+  (https://github.com/PyPSA/pypsa-eur/pull/732)
+
+* Fix nodal fraction in :mod:`add_existing_year` when using distributed
+  generators (https://github.com/PyPSA/pypsa-eur/pull/798).
+
+* Bugfix: District heating without progress caused division by zero
+  (https://github.com/PyPSA/pypsa-eur/pull/796).
+
+* Bugfix: Drop duplicates in :mod:`build_industrial_distribution_keys`, which
+  can occur through the geopandas ``.sjoin()`` function if a point is located on
+  a border (https://github.com/PyPSA/pypsa-eur/pull/726).
+
+* For network clustering fall back to ``ipopt`` when ``highs`` is designated
+  solver (https://github.com/PyPSA/pypsa-eur/pull/795).
+
+* Fix typo in buses definition for oil boilers in ``add_industry`` in
+  :mod:`prepare_sector_network` (https://github.com/PyPSA/pypsa-eur/pull/812).
+
+* Resolve code issues for endogenous building retrofitting. Select correct
+  sector names, address deprecations, distinguish between district heating,
+  decentral heating in urban areas or rural areas for floor area calculations
+  (https://github.com/PyPSA/pypsa-eur/pull/808).
+
+* Addressed various deprecations.


 PyPSA-Eur 0.8.1 (27th July 2023)
@ -166,6 +725,8 @@ PyPSA-Eur 0.8.1 (27th July 2023)
  (https://github.com/PyPSA/pypsa-eur/pull/672)


+* Addressed deprecation warnings for ``pandas=2.0``. ``pandas=2.0`` is now minimum requirement.
+
 PyPSA-Eur 0.8.0 (18th March 2023)
 =================================

@ -1427,8 +1988,4 @@ Release Process

 * Make a `GitHub release <https://github.com/PyPSA/pypsa-eur-sec/releases>`_, which automatically triggers archiving to the `zenodo code repository <https://doi.org/10.5281/zenodo.3520874>`_ with `MIT license <https://opensource.org/licenses/MIT>`_.

-* Create pre-built networks for ``config.default.yaml`` by running ``snakemake -call prepare_sector_networks``.
-
-* Upload pre-built networks to `zenodo data repository <https://doi.org/10.5281/zenodo.3601881>`_ with `CC BY 4.0 <https://creativecommons.org/licenses/by/4.0/>`_ license.
-
 * Send announcement on the `PyPSA mailing list <https://groups.google.com/forum/#!forum/pypsa>`_.
--- a/doc/requirements.txt
+++ b/doc/requirements.txt
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2019-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2019-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0

--- a/doc/retrieve.rst
+++ b/doc/retrieve.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

@ -22,11 +22,11 @@ Rule ``retrieve_databundle``
 Rule ``retrieve_cutout``
 ============================

-.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517949.svg
-   :target: https://doi.org/10.5281/zenodo.3517949
+.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.6382570.svg
+   :target: https://doi.org/10.5281/zenodo.6382570

 Cutouts are spatio-temporal subsets of the European weather data from the `ECMWF ERA5 <https://software.ecmwf.int/wiki/display/CKB/ERA5+data+documentation>`_ reanalysis dataset and the `CMSAF SARAH-2 <https://wui.cmsaf.eu/safira/action/viewDoiDetails?acronym=SARAH_V002>`_ solar surface radiation dataset for the year 2013.
-They have been prepared by and are for use with the `atlite <https://github.com/PyPSA/atlite>`_ tool. You can either generate them yourself using the ``build_cutouts`` rule or retrieve them directly from `zenodo <https://doi.org/10.5281/zenodo.3517949>`__ through the rule ``retrieve_cutout``.
+They have been prepared by and are for use with the `atlite <https://github.com/PyPSA/atlite>`_ tool. You can either generate them yourself using the ``build_cutouts`` rule or retrieve them directly from `zenodo <https://doi.org/10.5281/zenodo.6382570>`__ through the rule ``retrieve_cutout``.
 The :ref:`tutorial` uses a smaller cutout than required for the full model (30 MB), which is also automatically downloaded.

 .. note::
@ -91,7 +91,7 @@ None.

 **Outputs**

- ``data/load_raw.csv``
+- ``data/electricity_demand_raw.csv``


 Rule ``retrieve_cost_data``
--- a/doc/sector.rst
+++ b/doc/sector.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2023-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

@ -20,6 +20,12 @@ Rule ``add_existing_baseyear``

 .. automodule:: add_existing_baseyear

+Rule ``build_existing_heating_distribution``
+==============================================================================
+
+.. automodule:: build_existing_heating_distribution
+
+
 Rule ``build_ammonia_production``
 ==============================================================================

@ -60,10 +66,20 @@ Rule ``build_gas_network``

 .. automodule:: build_gas_network

-Rule ``build_heat_demand``
+Rule ``build_daily_heat_demand``
 ==============================================================================

-.. automodule:: build_heat_demand
+.. automodule:: build_daily_heat_demand
+
+Rule ``build_hourly_heat_demand``
+==============================================================================
+
+.. automodule:: build_hourly_heat_demand
+
+Rule ``build_district_heat_share``
+==============================================================================
+
+.. automodule:: build_district_heat_share

 Rule ``build_industrial_distribution_key``
 ==============================================================================
--- a/doc/simplification.rst
+++ b/doc/simplification.rst
@ -1,7 +1,7 @@


 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

--- a/doc/solving.rst
+++ b/doc/solving.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

--- a/doc/spatial_resolution.rst
+++ b/doc/spatial_resolution.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2021-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2021-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

@ -45,7 +45,7 @@ Here are some examples of how spatial resolution is set for different sectors in

 •	CO2: It can be modeled as a single node for Europe or it can be nodally resolved with CO2 transport pipelines if activated in the `config <https://github.com/PyPSA/pypsa-eur-sec/blob/3daff49c9999ba7ca7534df4e587e1d516044fc3/config.default.yaml#L248>`_. It should mentioned that in single node mode a transport and storage cost is added for sequestered CO2, the cost of which can be adjusted in the `config <https://github.com/PyPSA/pypsa-eur-sec/blob/3daff49c9999ba7ca7534df4e587e1d516044fc3/config.default.yaml#L247>`_.

-•	Liquid hydrocarbons: Modeled as a single node for Europe, since transport costs for liquids are low and no bottlenecks are expected.
+•	Carbonaceous fuels: Modeled as a single node for Europe by default, since transport costs for liquids are low and no bottlenecks are expected. Can be regionally resolved in configuration.

 **Electricity distribution network**

--- a/doc/supply_demand.rst
+++ b/doc/supply_demand.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2021-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2021-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

--- a/doc/support.rst
+++ b/doc/support.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

--- a/doc/tutorial.rst
+++ b/doc/tutorial.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

@ -25,7 +25,7 @@ full model, which allows the user to explore most of its functionalities on a
 local machine. The tutorial will cover examples on how to configure and
 customise the PyPSA-Eur model and run the ``snakemake`` workflow step by step
 from network creation to the solved network. The configuration for the tutorial
-is located at ``test/config.electricity.yaml``. It includes parts deviating from
+is located at ``config/test/config.electricity.yaml``. It includes parts deviating from
 the default config file ``config/config.default.yaml``. To run the tutorial with this
 configuration, execute

@ -96,7 +96,7 @@ open-source solver GLPK.
   :start-at: solver:
   :end-before: plotting:

-Note, that ``test/config.electricity.yaml`` only includes changes relative to
+Note, that ``config/test/config.electricity.yaml`` only includes changes relative to
 the default configuration. There are many more configuration options, which are
 documented at :ref:`config`.

@ -133,82 +133,89 @@ This triggers a workflow of multiple preceding jobs that depend on each rule's i
        graph[bgcolor=white, margin=0];
        node[shape=box, style=rounded, fontname=sans,                 fontsize=10, penwidth=2];
        edge[penwidth=2, color=grey];
-            0[label = "solve_network", color = "0.33 0.6 0.85", style="rounded"];
-            1[label = "prepare_network\nll: copt\nopts: Co2L-24H", color = "0.03 0.6 0.85", style="rounded"];
-            2[label = "add_extra_components", color = "0.45 0.6 0.85", style="rounded"];
-            3[label = "cluster_network\nclusters: 6", color = "0.46 0.6 0.85", style="rounded"];
-            4[label = "simplify_network\nsimpl: ", color = "0.52 0.6 0.85", style="rounded"];
-            5[label = "add_electricity", color = "0.55 0.6 0.85", style="rounded"];
-            6[label = "build_renewable_profiles\ntechnology: solar", color = "0.15 0.6 0.85", style="rounded"];
-            7[label = "base_network", color = "0.37 0.6 0.85", style="rounded,dashed"];
-            8[label = "build_shapes", color = "0.07 0.6 0.85", style="rounded,dashed"];
-            9[label = "retrieve_databundle", color = "0.60 0.6 0.85", style="rounded"];
-            10[label = "retrieve_natura_raster", color = "0.42 0.6 0.85", style="rounded"];
-            11[label = "build_bus_regions", color = "0.09 0.6 0.85", style="rounded,dashed"];
-            12[label = "build_renewable_profiles\ntechnology: onwind", color = "0.15 0.6 0.85", style="rounded"];
-            13[label = "build_renewable_profiles\ntechnology: offwind-ac", color = "0.15 0.6 0.85", style="rounded"];
-            14[label = "build_ship_raster", color = "0.02 0.6 0.85", style="rounded"];
-            15[label = "retrieve_ship_raster", color = "0.40 0.6 0.85", style="rounded"];
-            16[label = "build_renewable_profiles\ntechnology: offwind-dc", color = "0.15 0.6 0.85", style="rounded"];
-            17[label = "build_line_rating", color = "0.32 0.6 0.85", style="rounded"];
-            18[label = "retrieve_cost_data\nyear: 2030", color = "0.50 0.6 0.85", style="rounded"];
-            19[label = "build_powerplants", color = "0.64 0.6 0.85", style="rounded,dashed"];
-            20[label = "build_electricity_demand", color = "0.13 0.6 0.85", style="rounded,dashed"];
-            21[label = "retrieve_electricity_demand", color = "0.31 0.6 0.85", style="rounded"];
-            22[label = "copy_config", color = "0.23 0.6 0.85", style="rounded"];
+            0[label = "solve_network", color = "0.39 0.6 0.85", style="rounded"];
+            1[label = "prepare_network\nll: copt\nopts: Co2L-24H", color = "0.29 0.6 0.85", style="rounded"];
+            2[label = "add_extra_components", color = "0.28 0.6 0.85", style="rounded"];
+            3[label = "cluster_network\nclusters: 6", color = "0.19 0.6 0.85", style="rounded"];
+            4[label = "simplify_network\nsimpl: ", color = "0.01 0.6 0.85", style="rounded"];
+            5[label = "add_electricity", color = "0.49 0.6 0.85", style="rounded"];
+            6[label = "build_renewable_profiles\ntechnology: solar", color = "0.21 0.6 0.85", style="rounded"];
+            7[label = "base_network", color = "0.27 0.6 0.85", style="rounded"];
+            8[label = "build_shapes", color = "0.26 0.6 0.85", style="rounded"];
+            9[label = "retrieve_databundle", color = "0.59 0.6 0.85", style="rounded"];
+            10[label = "retrieve_natura_raster", color = "0.47 0.6 0.85", style="rounded"];
+            11[label = "build_bus_regions", color = "0.13 0.6 0.85", style="rounded"];
+            12[label = "retrieve_cutout\ncutout: be-03-2013-era5", color = "0.36 0.6 0.85", style="rounded,dashed"];
+            13[label = "build_renewable_profiles\ntechnology: onwind", color = "0.21 0.6 0.85", style="rounded"];
+            14[label = "build_renewable_profiles\ntechnology: offwind-ac", color = "0.21 0.6 0.85", style="rounded"];
+            15[label = "build_ship_raster", color = "0.00 0.6 0.85", style="rounded"];
+            16[label = "retrieve_ship_raster", color = "0.51 0.6 0.85", style="rounded,dashed"];
+            17[label = "build_renewable_profiles\ntechnology: offwind-dc", color = "0.21 0.6 0.85", style="rounded"];
+            18[label = "build_line_rating", color = "0.05 0.6 0.85", style="rounded"];
+            19[label = "retrieve_cost_data\nyear: 2030", color = "0.15 0.6 0.85", style="rounded"];
+            20[label = "build_powerplants", color = "0.54 0.6 0.85", style="rounded"];
+            21[label = "build_electricity_demand", color = "0.52 0.6 0.85", style="rounded"];
+            22[label = "retrieve_electricity_demand", color = "0.22 0.6 0.85", style="rounded"];
+            23[label = "copy_config", color = "0.44 0.6 0.85", style="rounded"];
            1 -> 0
-            22 -> 0
+            23 -> 0
            2 -> 1
-            18 -> 1
+            19 -> 1
            3 -> 2
-            18 -> 2
+            19 -> 2
            4 -> 3
-            18 -> 3
+            19 -> 3
            5 -> 4
-            18 -> 4
+            19 -> 4
            11 -> 4
            6 -> 5
-            12 -> 5
            13 -> 5
-            16 -> 5
-            7 -> 5
+            14 -> 5
            17 -> 5
+            7 -> 5
            18 -> 5
-            11 -> 5
            19 -> 5
-            9 -> 5
+            11 -> 5
            20 -> 5
+            9 -> 5
+            21 -> 5
            8 -> 5
            7 -> 6
            9 -> 6
            10 -> 6
            8 -> 6
            11 -> 6
+            12 -> 6
            8 -> 7
            9 -> 8
            8 -> 11
            7 -> 11
-            7 -> 12
-            9 -> 12
-            10 -> 12
-            8 -> 12
-            11 -> 12
            7 -> 13
            9 -> 13
            10 -> 13
-            14 -> 13
            8 -> 13
            11 -> 13
+            12 -> 13
+            7 -> 14
+            9 -> 14
+            10 -> 14
            15 -> 14
-            7 -> 16
-            9 -> 16
-            10 -> 16
-            14 -> 16
-            8 -> 16
-            11 -> 16
+            8 -> 14
+            11 -> 14
+            12 -> 14
+            16 -> 15
+            12 -> 15
            7 -> 17
-            7 -> 19
-            21 -> 20
+            9 -> 17
+            10 -> 17
+            15 -> 17
+            8 -> 17
+            11 -> 17
+            12 -> 17
+            7 -> 18
+            12 -> 18
+            7 -> 20
+            22 -> 21
    }

 |
@ -218,26 +225,29 @@ In the terminal, this will show up as a list of jobs to be run:
 .. code:: bash

    Building DAG of jobs...
-    job                         count    min threads    max threads
-    ------------------------  -------  -------------  -------------
-    add_electricity                 1              1              1
-    add_extra_components            1              1              1
-    base_network                    1              1              1
-    build_bus_regions               1              1              1
-    build_hydro_profile             1              1              1
-    build_electricity_demand        1              1              1
-    build_powerplants               1              1              1
-    build_renewable_profiles        4              1              1
-    build_shapes                    1              1              1
-    build_ship_raster               1              1              1
-    cluster_network                 1              1              1
-    prepare_network                 1              1              1
-    retrieve_cost_data              1              1              1
-    retrieve_databundle             1              1              1
-    retrieve_natura_raster          1              1              1
-    simplify_network                1              1              1
-    solve_network                   1              1              1
-    total                          20              1              1
+    Job stats:
+    job                            count
+    ---------------------------  -------
+    add_electricity                    1
+    add_extra_components               1
+    base_network                       1
+    build_bus_regions                  1
+    build_electricity_demand           1
+    build_line_rating                  1
+    build_powerplants                  1
+    build_renewable_profiles           4
+    build_shapes                       1
+    build_ship_raster                  1
+    cluster_network                    1
+    copy_config                        1
+    prepare_network                    1
+    retrieve_cost_data                 1
+    retrieve_databundle                1
+    retrieve_electricity_demand        1
+    retrieve_natura_raster             1
+    simplify_network                   1
+    solve_network                      1
+    total                             22


 ``snakemake`` then runs these jobs in the correct order.
@ -246,16 +256,16 @@ A job (here ``simplify_network``) will display its attributes and normally some

 .. code:: bash

-    [Mon Jan 1 00:00:00 2023]
+    [Mon Feb 19 17:06:17 2024]
    rule simplify_network:
-        input: networks/elec.nc, resources/costs.csv, resources/regions_onshore.geojson, resources/regions_offshore.geojson
-        output: networks/elec_s.nc, resources/regions_onshore_elec_s.geojson, resources/regions_offshore_elec_s.geojson, resources/busmap_elec_s.csv, resources/connection_costs_s.csv
-        log: logs/simplify_network/elec_s.log
+        input: resources/test/networks/elec.nc, data/costs_2030.csv, resources/test/regions_onshore.geojson, resources/test/regions_offshore.geojson
+        output: resources/test/networks/elec_s.nc, resources/test/regions_onshore_elec_s.geojson, resources/test/regions_offshore_elec_s.geojson, resources/test/busmap_elec_s.csv, resources/test/connection_costs_s.csv
+        log: logs/test-elec/simplify_network/elec_s.log
        jobid: 4
-        benchmark: benchmarks/simplify_network/elec_s
-        reason: Missing output files: resources/busmap_elec_s.csv, resources/regions_onshore_elec_s.geojson, networks/elec_s.nc, resources/regions_offshore_elec_s.geojson; Input files updated by another job: resources/regions_offshore.geojson, resources/regions_onshore.geojson, resources/costs.csv, networks/elec.nc
+        benchmark: benchmarks/test-elec/simplify_network/elec_s
+        reason: Missing output files: resources/test/regions_offshore_elec_s.geojson, resources/test/busmap_elec_s.csv, resources/test/regions_onshore_elec_s.geojson, resources/test/networks/elec_s.nc; Input files updated by another job: resources/test/regions_offshore.geojson, resources/test/networks/elec.nc, resources/test/regions_onshore.geojson, data/costs_2030.csv
        wildcards: simpl=
-        resources: tmpdir=/tmp, mem_mb=4000, mem_mib=3815
+        resources: tmpdir=/tmp, mem_mb=12000, mem_mib=11445

 Once the whole worktree is finished, it should state so in the terminal.

--- a/doc/tutorial_sector.rst
+++ b/doc/tutorial_sector.rst
--- a/doc/validation.rst
+++ b/doc/validation.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

--- a/doc/wildcards.rst
+++ b/doc/wildcards.rst
@ -1,5 +1,5 @@
 ..
-  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+  SPDX-FileCopyrightText: 2019-2024 The PyPSA-Eur Authors

  SPDX-License-Identifier: CC-BY-4.0

--- a/envs/environment.fixed.yaml
+++ b/envs/environment.fixed.yaml
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0

@ -12,94 +12,90 @@ dependencies:
 - _libgcc_mutex=0.1
 - _openmp_mutex=4.5
 - affine=2.4.0
- alsa-lib=1.2.9
+- alsa-lib=1.2.10
 - ampl-mp=3.1.0
 - amply=0.1.6
- anyio=3.7.1
 - appdirs=1.4.4
- argon2-cffi=21.3.0
- argon2-cffi-bindings=21.2.0
- asttokens=2.2.1
- async-lru=2.0.3
+- asttokens=2.4.1
 - atk-1.0=2.38.0
- atlite=0.2.11
+- atlite=0.2.12
 - attr=2.5.1
- attrs=23.1.0
- aws-c-auth=0.7.0
- aws-c-cal=0.6.0
- aws-c-common=0.8.23
+- attrs=23.2.0
+- aws-c-auth=0.7.15
+- aws-c-cal=0.6.9
+- aws-c-common=0.9.12
 - aws-c-compression=0.2.17
- aws-c-event-stream=0.3.1
- aws-c-http=0.7.11
- aws-c-io=0.13.28
- aws-c-mqtt=0.8.14
- aws-c-s3=0.3.13
- aws-c-sdkutils=0.1.11
- aws-checksums=0.1.16
- aws-crt-cpp=0.20.3
- aws-sdk-cpp=1.10.57
- babel=2.12.1
- backcall=0.2.0
- backports=1.0
- backports.functools_lru_cache=1.6.5
- beautifulsoup4=4.12.2
- bleach=6.0.0
- blosc=1.21.4
- bokeh=3.2.1
- boost-cpp=1.78.0
+- aws-c-event-stream=0.4.1
+- aws-c-http=0.8.0
+- aws-c-io=0.14.3
+- aws-c-mqtt=0.10.1
+- aws-c-s3=0.5.0
+- aws-c-sdkutils=0.1.14
+- aws-checksums=0.1.17
+- aws-crt-cpp=0.26.1
+- aws-sdk-cpp=1.11.242
+- azure-core-cpp=1.10.3
+- azure-storage-blobs-cpp=12.10.0
+- azure-storage-common-cpp=12.5.0
+- beautifulsoup4=4.12.3
+- blosc=1.21.5
+- bokeh=3.3.4
 - bottleneck=1.3.7
- branca=0.6.0
- brotli=1.0.9
- brotli-bin=1.0.9
- brotli-python=1.0.9
+- branca=0.7.1
+- brotli=1.1.0
+- brotli-bin=1.1.0
+- brotli-python=1.1.0
 - bzip2=1.0.8
- c-ares=1.19.1
- c-blosc2=2.10.0
- ca-certificates=2023.7.22
- cairo=1.16.0
- cartopy=0.21.1
+- c-ares=1.26.0
+- c-blosc2=2.13.2
+- ca-certificates=2024.2.2
+- cairo=1.18.0
+- cartopy=0.22.0
 - cdsapi=0.6.1
- certifi=2023.7.22
- cffi=1.15.1
- cfitsio=4.2.0
- cftime=1.6.2
- charset-normalizer=3.2.0
- click=8.1.6
+- certifi=2024.2.2
+- cffi=1.16.0
+- cfgv=3.3.1
+- cfitsio=4.3.1
+- cftime=1.6.3
+- charset-normalizer=3.3.2
+- click=8.1.7
 - click-plugins=1.1.1
 - cligj=0.7.2
- cloudpickle=2.2.1
+- cloudpickle=3.0.0
+- coin-or-cbc=2.10.10
+- coin-or-cgl=0.60.7
+- coin-or-clp=1.17.8
+- coin-or-osi=0.108.8
+- coin-or-utils=2.11.9
+- coincbc=2.10.10
 - colorama=0.4.6
- comm=0.1.3
 - configargparse=1.7
 - connection_pool=0.0.3
- contourpy=1.1.0
- country_converter=1.0.0
- curl=8.2.0
- cycler=0.11.0
- cytoolz=0.12.2
- dask=2023.7.1
- dask-core=2023.7.1
+- contourpy=1.2.0
+- country_converter=1.2
+- cppad=20240000.2
+- cycler=0.12.1
+- cytoolz=0.12.3
+- dask=2024.2.0
+- dask-core=2024.2.0
 - datrie=0.8.2
 - dbus=1.13.6
- debugpy=1.6.7
 - decorator=5.1.1
- defusedxml=0.7.1
 - deprecation=2.1.0
 - descartes=1.1.0
- distributed=2023.7.1
- distro=1.8.0
+- distlib=0.3.8
+- distributed=2024.2.0
+- distro=1.9.0
 - docutils=0.20.1
 - dpath=2.1.6
- entrypoints=0.4
- entsoe-py=0.5.10
+- entsoe-py=0.6.6
 - et_xmlfile=1.1.0
- exceptiongroup=1.1.2
- executing=1.2.0
+- exceptiongroup=1.2.0
+- executing=2.0.1
 - expat=2.5.0
- filelock=3.12.2
- fiona=1.9.4
- flit-core=3.9.0
- folium=0.14.0
+- filelock=3.13.1
+- fiona=1.9.5
+- folium=0.15.1
 - font-ttf-dejavu-sans-mono=2.37
 - font-ttf-inconsolata=3.000
 - font-ttf-source-code-pro=2.038
@ -107,366 +103,344 @@ dependencies:
 - fontconfig=2.14.2
 - fonts-conda-ecosystem=1
 - fonts-conda-forge=1
- fonttools=4.41.1
+- fonttools=4.49.0
 - freetype=2.12.1
- freexl=1.0.6
+- freexl=2.0.0
 - fribidi=1.0.10
- fsspec=2023.6.0
- gdal=3.7.0
+- fsspec=2024.2.0
+- gdal=3.8.4
 - gdk-pixbuf=2.42.10
 - geographiclib=1.52
- geojson-rewind=1.0.2
- geopandas=0.13.2
- geopandas-base=0.13.2
- geopy=2.3.0
- geos=3.11.2
+- geojson-rewind=1.1.0
+- geopandas=0.14.3
+- geopandas-base=0.14.3
+- geopy=2.4.1
+- geos=3.12.1
 - geotiff=1.7.1
 - gettext=0.21.1
 - gflags=2.2.2
 - giflib=5.2.1
- gitdb=4.0.10
- gitpython=3.1.32
- glib=2.76.4
- glib-tools=2.76.4
+- gitdb=4.0.11
+- gitpython=3.1.42
+- glib=2.78.4
+- glib-tools=2.78.4
 - glog=0.6.0
- gmp=6.2.1
+- glpk=5.0
+- gmp=6.3.0
 - graphite2=1.3.13
- graphviz=8.1.0
- gst-plugins-base=1.22.5
- gstreamer=1.22.5
+- graphviz=9.0.0
+- gst-plugins-base=1.22.9
+- gstreamer=1.22.9
 - gtk2=2.24.33
 - gts=0.7.6
- harfbuzz=7.3.0
+- harfbuzz=8.3.0
 - hdf4=4.2.15
- hdf5=1.14.1
+- hdf5=1.14.3
 - humanfriendly=10.0
- icu=72.1
- idna=3.4
- importlib-metadata=6.8.0
- importlib_metadata=6.8.0
- importlib_resources=6.0.0
+- icu=73.2
+- identify=2.5.35
+- idna=3.6
+- importlib-metadata=7.0.1
+- importlib_metadata=7.0.1
+- importlib_resources=6.1.1
 - iniconfig=2.0.0
- ipopt=3.14.12
- ipykernel=6.24.0
- ipython=8.14.0
- ipython_genutils=0.2.0
- ipywidgets=8.0.7
- jedi=0.18.2
- jinja2=3.1.2
- joblib=1.3.0
- json-c=0.16
- json5=0.9.14
- jsonschema=4.18.4
- jsonschema-specifications=2023.7.1
- jupyter=1.0.0
- jupyter-lsp=2.2.0
- jupyter_client=8.3.0
- jupyter_console=6.6.3
- jupyter_core=5.3.1
- jupyter_events=0.6.3
- jupyter_server=2.7.0
- jupyter_server_terminals=0.4.4
- jupyterlab=4.0.3
- jupyterlab_pygments=0.2.2
- jupyterlab_server=2.24.0
- jupyterlab_widgets=3.0.8
- kealib=1.5.1
+- ipopt=3.14.14
+- ipython=8.21.0
+- jedi=0.19.1
+- jinja2=3.1.3
+- joblib=1.3.2
+- json-c=0.17
+- jsonschema=4.21.1
+- jsonschema-specifications=2023.12.1
+- jupyter_core=5.7.1
+- kealib=1.5.3
 - keyutils=1.6.1
- kiwisolver=1.4.4
- krb5=1.21.1
+- kiwisolver=1.4.5
+- krb5=1.21.2
 - lame=3.100
- lcms2=2.15
+- lcms2=2.16
 - ld_impl_linux-64=2.40
 - lerc=4.0.0
- libabseil=20230125.3
- libaec=1.0.6
- libarchive=3.6.2
- libarrow=12.0.1
+- libabseil=20230802.1
+- libaec=1.1.2
+- libarchive=3.7.2
+- libarrow=15.0.0
+- libarrow-acero=15.0.0
+- libarrow-dataset=15.0.0
+- libarrow-flight=15.0.0
+- libarrow-flight-sql=15.0.0
+- libarrow-gandiva=15.0.0
+- libarrow-substrait=15.0.0
 - libblas=3.9.0
- libbrotlicommon=1.0.9
- libbrotlidec=1.0.9
- libbrotlienc=1.0.9
- libcap=2.67
+- libboost-headers=1.84.0
+- libbrotlicommon=1.1.0
+- libbrotlidec=1.1.0
+- libbrotlienc=1.1.0
+- libcap=2.69
 - libcblas=3.9.0
 - libclang=15.0.7
 - libclang13=15.0.7
 - libcrc32c=1.1.2
 - libcups=2.3.3
- libcurl=8.2.0
- libdeflate=1.18
+- libcurl=8.5.0
+- libdeflate=1.19
 - libedit=3.1.20191231
 - libev=4.33
 - libevent=2.1.12
 - libexpat=2.5.0
 - libffi=3.4.2
 - libflac=1.4.3
- libgcc-ng=13.1.0
- libgcrypt=1.10.1
+- libgcc-ng=13.2.0
+- libgcrypt=1.10.3
 - libgd=2.3.3
- libgdal=3.7.0
- libgfortran-ng=13.1.0
- libgfortran5=13.1.0
- libglib=2.76.4
- libgomp=13.1.0
+- libgdal=3.8.4
+- libgfortran-ng=13.2.0
+- libgfortran5=13.2.0
+- libglib=2.78.4
+- libgomp=13.2.0
 - libgoogle-cloud=2.12.0
 - libgpg-error=1.47
- libgrpc=1.56.2
+- libgrpc=1.60.1
+- libhwloc=2.9.3
 - libiconv=1.17
- libjpeg-turbo=2.1.5.1
+- libjpeg-turbo=3.0.0
 - libkml=1.3.0
 - liblapack=3.9.0
 - liblapacke=3.9.0
 - libllvm15=15.0.7
 - libnetcdf=4.9.2
- libnghttp2=1.52.0
- libnsl=2.0.0
+- libnghttp2=1.58.0
+- libnl=3.9.0
+- libnsl=2.0.1
 - libnuma=2.0.16
 - libogg=1.3.4
- libopenblas=0.3.23
+- libopenblas=0.3.26
 - libopus=1.3.1
- libpng=1.6.39
- libpq=15.3
- libprotobuf=4.23.3
- librsvg=2.56.1
+- libparquet=15.0.0
+- libpng=1.6.42
+- libpq=16.2
+- libprotobuf=4.25.1
+- libre2-11=2023.06.02
+- librsvg=2.56.3
 - librttopo=1.1.0
- libsndfile=1.2.0
- libsodium=1.0.18
+- libscotch=7.0.4
+- libsndfile=1.2.2
 - libspatialindex=1.9.3
- libspatialite=5.0.1
- libsqlite=3.42.0
+- libspatialite=5.1.0
+- libspral=2023.09.07
+- libsqlite=3.45.1
 - libssh2=1.11.0
- libstdcxx-ng=13.1.0
- libsystemd0=253
- libthrift=0.18.1
- libtiff=4.5.1
- libtool=2.4.7
+- libstdcxx-ng=13.2.0
+- libsystemd0=255
+- libthrift=0.19.0
+- libtiff=4.6.0
 - libutf8proc=2.8.0
 - libuuid=2.38.1
 - libvorbis=1.3.7
- libwebp=1.3.1
- libwebp-base=1.3.1
+- libwebp=1.3.2
+- libwebp-base=1.3.2
 - libxcb=1.15
- libxkbcommon=1.5.0
- libxml2=2.11.4
- libxslt=1.1.37
- libzip=1.9.2
+- libxcrypt=4.4.36
+- libxkbcommon=1.6.0
+- libxml2=2.12.5
+- libxslt=1.1.39
+- libzip=1.10.1
 - libzlib=1.2.13
+- linopy=0.3.4
 - locket=1.0.0
- lxml=4.9.3
- lz4=4.3.2
+- lxml=5.1.0
+- lz4=4.3.3
 - lz4-c=1.9.4
 - lzo=2.10
- mapclassify=2.5.0
- markupsafe=2.1.3
- matplotlib=3.5.3
- matplotlib-base=3.5.3
+- mapclassify=2.6.1
+- markupsafe=2.1.5
+- matplotlib=3.8.3
+- matplotlib-base=3.8.3
 - matplotlib-inline=0.1.6
 - memory_profiler=0.61.0
- metis=5.1.1
- mistune=3.0.0
- mpg123=1.31.3
- msgpack-python=1.0.5
- mumps-include=5.2.1
- mumps-seq=5.2.1
- munch=4.0.0
+- metis=5.1.0
+- minizip=4.0.4
+- mpg123=1.32.4
+- msgpack-python=1.0.7
+- mumps-include=5.6.2
+- mumps-seq=5.6.2
 - munkres=1.1.4
 - mysql-common=8.0.33
 - mysql-libs=8.0.33
- nbclient=0.8.0
- nbconvert=7.7.2
- nbconvert-core=7.7.2
- nbconvert-pandoc=7.7.2
- nbformat=5.9.1
+- nbformat=5.9.2
 - ncurses=6.4
- nest-asyncio=1.5.6
- netcdf4=1.6.4
- networkx=3.1
+- netcdf4=1.6.5
+- networkx=3.2.1
+- nodeenv=1.8.0
 - nomkl=1.0
- notebook=7.0.0
- notebook-shim=0.2.3
 - nspr=4.35
- nss=3.89
- numexpr=2.8.4
- numpy=1.25.1
- openjdk=17.0.3
+- nss=3.98
+- numexpr=2.9.0
+- numpy=1.26.4
+- openjdk=21.0.2
 - openjpeg=2.5.0
 - openpyxl=3.1.2
- openssl=3.1.1
- orc=1.9.0
- overrides=7.3.1
- packaging=23.1
- pandas=2.0.3
- pandoc=3.1.3
- pandocfilters=1.5.0
+- openssl=3.2.1
+- orc=1.9.2
+- packaging=23.2
+- pandas=2.2.0
 - pango=1.50.14
 - parso=0.8.3
- partd=1.4.0
- patsy=0.5.3
- pcre2=10.40
- pexpect=4.8.0
+- partd=1.4.1
+- patsy=0.5.6
+- pcre2=10.42
+- pexpect=4.9.0
 - pickleshare=0.7.5
- pillow=10.0.0
- pip=23.2.1
- pixman=0.40.0
+- pillow=10.2.0
+- pip=24.0
+- pixman=0.43.2
 - pkgutil-resolve-name=1.3.10
- plac=1.3.5
- platformdirs=3.9.1
- pluggy=1.2.0
+- plac=1.4.2
+- platformdirs=4.2.0
+- pluggy=1.4.0
 - ply=3.11
- pooch=1.7.0
- poppler=23.05.0
+- poppler=24.02.0
 - poppler-data=0.4.12
- postgresql=15.3
- powerplantmatching=0.5.7
- progressbar2=4.2.0
- proj=9.2.1
- prometheus_client=0.17.1
- prompt-toolkit=3.0.39
- prompt_toolkit=3.0.39
- psutil=5.9.5
+- postgresql=16.2
+- powerplantmatching=0.5.11
+- pre-commit=3.6.2
+- progressbar2=4.3.2
+- proj=9.3.1
+- prompt-toolkit=3.0.42
+- psutil=5.9.8
 - pthread-stubs=0.4
 - ptyprocess=0.7.0
 - pulp=2.7.0
 - pulseaudio-client=16.1
 - pure_eval=0.2.2
 - py-cpuinfo=9.0.0
- pyarrow=12.0.1
+- pyarrow=15.0.0
+- pyarrow-hotfix=0.6
 - pycountry=22.3.5
 - pycparser=2.21
- pygments=2.15.1
+- pygments=2.17.2
 - pyomo=6.6.1
- pyparsing=3.1.0
- pyproj=3.6.0
- pyqt=5.15.7
- pyqt5-sip=12.11.0
+- pyparsing=3.1.1
+- pyproj=3.6.1
+- pypsa=0.27.0
+- pyqt=5.15.9
+- pyqt5-sip=12.12.2
+- pyscipopt=4.4.0
 - pyshp=2.3.1
 - pysocks=1.7.1
- pytables=3.8.0
- pytest=7.4.0
- python=3.10.12
+- pytables=3.9.2
+- pytest=8.0.0
+- python=3.11.8
 - python-dateutil=2.8.2
- python-fastjsonschema=2.18.0
- python-json-logger=2.0.7
- python-tzdata=2023.3
- python-utils=3.7.0
- python_abi=3.10
- pytz=2023.3
+- python-fastjsonschema=2.19.1
+- python-tzdata=2024.1
+- python-utils=3.8.2
+- python_abi=3.11
+- pytz=2024.1
 - pyxlsb=1.0.10
- pyyaml=6.0
- pyzmq=25.1.0
+- pyyaml=6.0.1
 - qt-main=5.15.8
- qtconsole=5.4.3
- qtconsole-base=5.4.3
- qtpy=2.3.1
- rasterio=1.3.8
- rdma-core=28.9
- re2=2023.03.02
+- rasterio=1.3.9
+- rdma-core=50.0
+- re2=2023.06.02
 - readline=8.2
- referencing=0.30.0
+- referencing=0.33.0
 - requests=2.31.0
 - reretry=0.11.8
- rfc3339-validator=0.1.4
- rfc3986-validator=0.1.1
- rioxarray=0.14.1
- rpds-py=0.9.2
- rtree=1.0.1
- s2n=1.3.46
- scikit-learn=1.3.0
- scipy=1.11.1
- scotch=6.0.9
- seaborn=0.12.2
- seaborn-base=0.12.2
- send2trash=1.8.2
- setuptools=68.0.0
- setuptools-scm=7.1.0
- setuptools_scm=7.1.0
- shapely=2.0.1
- sip=6.7.10
+- rioxarray=0.15.1
+- rpds-py=0.18.0
+- rtree=1.2.0
+- s2n=1.4.3
+- scikit-learn=1.4.1.post1
+- scip=8.1.0
+- scipy=1.12.0
+- scotch=7.0.4
+- seaborn=0.13.2
+- seaborn-base=0.13.2
+- setuptools=69.1.0
+- setuptools-scm=8.0.4
+- setuptools_scm=8.0.4
+- shapely=2.0.2
+- sip=6.7.12
 - six=1.16.0
- smart_open=6.3.0
- smmap=3.0.5
- snakemake-minimal=7.30.2
+- smart_open=6.4.0
+- smmap=5.0.0
+- snakemake-minimal=7.32.4
 - snappy=1.1.10
- sniffio=1.3.0
 - snuggs=1.4.7
 - sortedcontainers=2.4.0
- soupsieve=2.3.2.post1
- sqlite=3.42.0
+- soupsieve=2.5
+- sqlite=3.45.1
 - stack_data=0.6.2
- statsmodels=0.14.0
+- statsmodels=0.14.1
 - stopit=1.1.2
- tabula-py=2.6.0
+- tabula-py=2.7.0
 - tabulate=0.9.0
- tblib=1.7.0
- terminado=0.17.1
- threadpoolctl=3.2.0
- throttler=1.2.1
- tiledb=2.13.2
- tinycss2=1.2.1
- tk=8.6.12
+- tbb=2021.11.0
+- tblib=3.0.0
+- threadpoolctl=3.3.0
+- throttler=1.2.2
+- tiledb=2.20.0
+- tk=8.6.13
 - toml=0.10.2
 - tomli=2.0.1
- toolz=0.12.0
+- toolz=0.12.1
 - toposort=1.10
- tornado=6.3.2
- tqdm=4.65.0
- traitlets=5.9.0
- typing-extensions=4.7.1
- typing_extensions=4.7.1
- typing_utils=0.1.0
- tzcode=2023c
- tzdata=2023c
- ucx=1.14.1
- unicodedata2=15.0.0
- unidecode=1.3.6
- unixodbc=2.3.10
- urllib3=2.0.4
- wcwidth=0.2.6
- webencodings=0.5.1
- websocket-client=1.6.1
- wheel=0.41.0
- widgetsnbextension=4.0.8
- wrapt=1.15.0
- xarray=2023.7.0
+- tornado=6.3.3
+- tqdm=4.66.2
+- traitlets=5.14.1
+- typing-extensions=4.9.0
+- typing_extensions=4.9.0
+- tzcode=2024a
+- tzdata=2024a
+- ucx=1.15.0
+- ukkonen=1.0.1
+- unidecode=1.3.8
+- unixodbc=2.3.12
+- uriparser=0.9.7
+- urllib3=2.2.1
+- validators=0.22.0
+- virtualenv=20.25.0
+- wcwidth=0.2.13
+- wheel=0.42.0
+- wrapt=1.16.0
+- xarray=2024.2.0
 - xcb-util=0.4.0
 - xcb-util-image=0.4.0
 - xcb-util-keysyms=0.4.0
 - xcb-util-renderutil=0.3.9
 - xcb-util-wm=0.4.1
- xerces-c=3.2.4
- xkeyboard-config=2.39
+- xerces-c=3.2.5
+- xkeyboard-config=2.41
 - xlrd=2.0.1
 - xorg-fixesproto=5.0
 - xorg-inputproto=2.3.2
 - xorg-kbproto=1.0.7
 - xorg-libice=1.1.1
 - xorg-libsm=1.2.4
- xorg-libx11=1.8.6
+- xorg-libx11=1.8.7
 - xorg-libxau=1.0.11
 - xorg-libxdmcp=1.1.3
 - xorg-libxext=1.3.4
 - xorg-libxfixes=5.0.3
 - xorg-libxi=1.7.10
 - xorg-libxrender=0.9.11
+- xorg-libxt=1.3.0
 - xorg-libxtst=1.2.3
 - xorg-recordproto=1.14.2
 - xorg-renderproto=0.11.1
 - xorg-xextproto=7.3.0
 - xorg-xf86vidmodeproto=2.3.1
 - xorg-xproto=7.0.31
- xyzservices=2023.7.0
+- xyzservices=2023.10.1
 - xz=5.2.6
 - yaml=0.2.5
- yte=1.5.1
- zeromq=4.3.4
+- yte=1.5.4
 - zict=3.0.0
- zipp=3.16.2
+- zipp=3.17.0
 - zlib=1.2.13
 - zlib-ng=2.0.7
- zstd=1.5.2
+- zstd=1.5.5
 - pip:
-  - gurobipy==10.0.2
-  - linopy==0.2.2
-  - pypsa==0.25.1
-  - tsam==2.3.0
-  - validators==0.20.0
+  - highspy==1.5.3
+  - tsam==2.3.1
--- a/envs/environment.yaml
+++ b/envs/environment.yaml
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT

@ -11,6 +11,8 @@ dependencies:
 - pip

 - atlite>=0.2.9
+- pypsa>=0.26.1
+- linopy
 - dask

  # Dependencies of the workflow itself
@ -18,23 +20,24 @@ dependencies:
 - openpyxl!=3.1.1
 - pycountry
 - seaborn
- snakemake-minimal>=7.7.0
+- snakemake-minimal>=8.5
 - memory_profiler
 - yaml
 - pytables
 - lxml
- powerplantmatching>=0.5.5
+- powerplantmatching>=0.5.5,!=0.5.9
 - numpy
- pandas>=1.4
+- pandas>=2.1
 - geopandas>=0.11.0
- xarray<=2023.8.0
+- xarray>=2023.11.0
 - rioxarray
 - netcdf4
 - networkx
 - scipy
+- glpk
 - shapely>=2.0
- pyomo
- matplotlib<3.6
+- pyscipopt
+- matplotlib
 - proj
 - fiona
 - country_converter
@ -44,6 +47,7 @@ dependencies:
 - tabula-py
 - pyxlsb
 - graphviz
+- pre-commit

  # Keep in conda environment when calling ipython
 - ipython
@ -56,4 +60,7 @@ dependencies:

 - pip:
  - tsam>=2.3.1
-  - pypsa>=0.25.2
+  - snakemake-storage-plugin-http
+  - snakemake-executor-plugin-slurm
+  - snakemake-executor-plugin-cluster-generic
+  - highspy
--- a/envs/retrieve.yaml
+++ b/envs/retrieve.yaml
@ -0,0 +1,18 @@
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+name: pypsa-eur-retrieve
+channels:
+- conda-forge
+- bioconda
+dependencies:
+- python>=3.8
+- pip
+- snakemake-minimal>=8.5
+- pandas>=2.1
+- tqdm
+- pip:
+  - snakemake-storage-plugin-http
+  - snakemake-executor-plugin-slurm
+  - snakemake-executor-plugin-cluster-generic
--- a/graphics/workflow.png
+++ b/graphics/workflow.png
--- a/2
+++ b/2
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: CC0-1.0
 font.family: sans-serif
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2023-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT

@ -8,7 +8,7 @@ if config["enable"].get("prepare_links_p_nom", False):
        output:
            "data/links_p_nom.csv",
        log:
-            LOGS + "prepare_links_p_nom.log",
+            logs("prepare_links_p_nom.log"),
        threads: 1
        resources:
            mem_mb=1500,
@ -20,15 +20,15 @@ if config["enable"].get("prepare_links_p_nom", False):

 rule build_electricity_demand:
    params:
-        snapshots=config["snapshots"],
-        countries=config["countries"],
-        load=config["load"],
+        snapshots=config_provider("snapshots"),
+        countries=config_provider("countries"),
+        load=config_provider("load"),
    input:
-        ancient("data/load_raw.csv"),
+        ancient("data/electricity_demand_raw.csv"),
    output:
-        RESOURCES + "load.csv",
+        resources("electricity_demand.csv"),
    log:
-        LOGS + "build_electricity_demand.log",
+        logs("build_electricity_demand.log"),
    resources:
        mem_mb=5000,
    conda:
@ -39,16 +39,17 @@ rule build_electricity_demand:

 rule build_powerplants:
    params:
-        powerplants_filter=config["electricity"]["powerplants_filter"],
-        custom_powerplants=config["electricity"]["custom_powerplants"],
-        countries=config["countries"],
+        powerplants_filter=config_provider("electricity", "powerplants_filter"),
+        custom_powerplants=config_provider("electricity", "custom_powerplants"),
+        everywhere_powerplants=config_provider("electricity", "everywhere_powerplants"),
+        countries=config_provider("countries"),
    input:
-        base_network=RESOURCES + "networks/base.nc",
+        base_network=resources("networks/base.nc"),
        custom_powerplants="data/custom_powerplants.csv",
    output:
-        RESOURCES + "powerplants.csv",
+        resources("powerplants.csv"),
    log:
-        LOGS + "build_powerplants.log",
+        logs("build_powerplants.log"),
    threads: 1
    resources:
        mem_mb=5000,
@ -60,11 +61,11 @@ rule build_powerplants:

 rule base_network:
    params:
-        countries=config["countries"],
-        snapshots=config["snapshots"],
-        lines=config["lines"],
-        links=config["links"],
-        transformers=config["transformers"],
+        countries=config_provider("countries"),
+        snapshots=config_provider("snapshots"),
+        lines=config_provider("lines"),
+        links=config_provider("links"),
+        transformers=config_provider("transformers"),
    input:
        eg_buses="data/entsoegridkit/buses.csv",
        eg_lines="data/entsoegridkit/lines.csv",
@ -74,15 +75,15 @@ rule base_network:
        parameter_corrections="data/parameter_corrections.yaml",
        links_p_nom="data/links_p_nom.csv",
        links_tyndp="data/links_tyndp.csv",
-        country_shapes=RESOURCES + "country_shapes.geojson",
-        offshore_shapes=RESOURCES + "offshore_shapes.geojson",
-        europe_shape=RESOURCES + "europe_shape.geojson",
+        country_shapes=resources("country_shapes.geojson"),
+        offshore_shapes=resources("offshore_shapes.geojson"),
+        europe_shape=resources("europe_shape.geojson"),
    output:
-        RESOURCES + "networks/base.nc",
+        resources("networks/base.nc"),
    log:
-        LOGS + "base_network.log",
+        logs("base_network.log"),
    benchmark:
-        BENCHMARKS + "base_network"
+        benchmarks("base_network")
    threads: 1
    resources:
        mem_mb=1500,
@ -94,7 +95,7 @@ rule base_network:

 rule build_shapes:
    params:
-        countries=config["countries"],
+        countries=config_provider("countries"),
    input:
        naturalearth=ancient("data/bundle/naturalearth/ne_10m_admin_0_countries.shp"),
        eez=ancient("data/bundle/eez/World_EEZ_v8_2014.shp"),
@ -104,12 +105,12 @@ rule build_shapes:
        ch_cantons=ancient("data/bundle/ch_cantons.csv"),
        ch_popgdp=ancient("data/bundle/je-e-21.03.02.xls"),
    output:
-        country_shapes=RESOURCES + "country_shapes.geojson",
-        offshore_shapes=RESOURCES + "offshore_shapes.geojson",
-        europe_shape=RESOURCES + "europe_shape.geojson",
-        nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
+        country_shapes=resources("country_shapes.geojson"),
+        offshore_shapes=resources("offshore_shapes.geojson"),
+        europe_shape=resources("europe_shape.geojson"),
+        nuts3_shapes=resources("nuts3_shapes.geojson"),
    log:
-        LOGS + "build_shapes.log",
+        logs("build_shapes.log"),
    threads: 1
    resources:
        mem_mb=1500,
@ -121,16 +122,16 @@ rule build_shapes:

 rule build_bus_regions:
    params:
-        countries=config["countries"],
+        countries=config_provider("countries"),
    input:
-        country_shapes=RESOURCES + "country_shapes.geojson",
-        offshore_shapes=RESOURCES + "offshore_shapes.geojson",
-        base_network=RESOURCES + "networks/base.nc",
+        country_shapes=resources("country_shapes.geojson"),
+        offshore_shapes=resources("offshore_shapes.geojson"),
+        base_network=resources("networks/base.nc"),
    output:
-        regions_onshore=RESOURCES + "regions_onshore.geojson",
-        regions_offshore=RESOURCES + "regions_offshore.geojson",
+        regions_onshore=resources("regions_onshore.geojson"),
+        regions_offshore=resources("regions_offshore.geojson"),
    log:
-        LOGS + "build_bus_regions.log",
+        logs("build_bus_regions.log"),
    threads: 1
    resources:
        mem_mb=1000,
@ -144,20 +145,20 @@ if config["enable"].get("build_cutout", False):

    rule build_cutout:
        params:
-            snapshots=config["snapshots"],
-            cutouts=config["atlite"]["cutouts"],
+            snapshots=config_provider("snapshots"),
+            cutouts=config_provider("atlite", "cutouts"),
        input:
-            regions_onshore=RESOURCES + "regions_onshore.geojson",
-            regions_offshore=RESOURCES + "regions_offshore.geojson",
+            regions_onshore=resources("regions_onshore.geojson"),
+            regions_offshore=resources("regions_offshore.geojson"),
        output:
            protected("cutouts/" + CDIR + "{cutout}.nc"),
        log:
-            "logs/" + CDIR + "build_cutout/{cutout}.log",
+            logs(CDIR + "build_cutout/{cutout}.log"),
        benchmark:
            "benchmarks/" + CDIR + "build_cutout_{cutout}"
-        threads: ATLITE_NPROCESSES
+        threads: config["atlite"].get("nprocesses", 4)
        resources:
-            mem_mb=ATLITE_NPROCESSES * 1000,
+            mem_mb=config["atlite"].get("nprocesses", 4) * 1000,
        conda:
            "../envs/environment.yaml"
        script:
@ -169,13 +170,15 @@ if config["enable"].get("build_natura_raster", False):
    rule build_natura_raster:
        input:
            natura=ancient("data/bundle/natura/Natura2000_end2015.shp"),
-            cutouts=expand("cutouts/" + CDIR + "{cutouts}.nc", **config["atlite"]),
+            cutouts=lambda w: expand(
+                "cutouts/" + CDIR + "{cutouts}.nc", **config_provider("atlite")(w)
+            ),
        output:
-            RESOURCES + "natura.tiff",
+            resources("natura.tiff"),
        resources:
            mem_mb=5000,
        log:
-            LOGS + "build_natura_raster.log",
+            logs("build_natura_raster.log"),
        conda:
            "../envs/environment.yaml"
        script:
@ -185,70 +188,129 @@ if config["enable"].get("build_natura_raster", False):
 rule build_ship_raster:
    input:
        ship_density="data/shipdensity_global.zip",
-        cutouts=expand(
+        cutouts=lambda w: expand(
            "cutouts/" + CDIR + "{cutout}.nc",
            cutout=[
-                config["renewable"][k]["cutout"]
-                for k in config["electricity"]["renewable_carriers"]
+                config_provider("renewable", k, "cutout")(w)
+                for k in config_provider("electricity", "renewable_carriers")(w)
            ],
        ),
    output:
-        RESOURCES + "shipdensity_raster.tif",
+        resources("shipdensity_raster.tif"),
    log:
-        LOGS + "build_ship_raster.log",
+        logs("build_ship_raster.log"),
    resources:
        mem_mb=5000,
    benchmark:
-        BENCHMARKS + "build_ship_raster"
+        benchmarks("build_ship_raster")
    conda:
        "../envs/environment.yaml"
    script:
        "../scripts/build_ship_raster.py"


+rule determine_availability_matrix_MD_UA:
+    input:
+        copernicus="data/Copernicus_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif",
+        wdpa="data/WDPA.gpkg",
+        wdpa_marine="data/WDPA_WDOECM_marine.gpkg",
+        gebco=lambda w: (
+            "data/bundle/GEBCO_2014_2D.nc"
+            if config_provider("renewable", w.technology)(w).get("max_depth")
+            else []
+        ),
+        ship_density=lambda w: (
+            resources("shipdensity_raster.tif")
+            if "ship_threshold" in config_provider("renewable", w.technology)(w).keys()
+            else []
+        ),
+        country_shapes=resources("country_shapes.geojson"),
+        offshore_shapes=resources("offshore_shapes.geojson"),
+        regions=lambda w: (
+            resources("regions_onshore.geojson")
+            if w.technology in ("onwind", "solar")
+            else resources("regions_offshore.geojson")
+        ),
+        cutout=lambda w: "cutouts/"
+        + CDIR
+        + config_provider("renewable", w.technology, "cutout")(w)
+        + ".nc",
+    output:
+        availability_matrix=resources("availability_matrix_MD-UA_{technology}.nc"),
+        availability_map=resources("availability_matrix_MD-UA_{technology}.png"),
+    log:
+        logs("determine_availability_matrix_MD_UA_{technology}.log"),
+    threads: config["atlite"].get("nprocesses", 4)
+    resources:
+        mem_mb=config["atlite"].get("nprocesses", 4) * 5000,
+    conda:
+        "../envs/environment.yaml"
+    script:
+        "../scripts/determine_availability_matrix_MD_UA.py"
+
+
+# Optional input when having Ukraine (UA) or Moldova (MD) in the countries list
+def input_ua_md_availability_matrix(w):
+    countries = set(config_provider("countries")(w))
+    if {"UA", "MD"}.intersection(countries):
+        return {
+            "availability_matrix_MD_UA": resources(
+                "availability_matrix_MD-UA_{technology}.nc"
+            )
+        }
+    return {}
+
+
 rule build_renewable_profiles:
    params:
-        renewable=config["renewable"],
+        snapshots=config_provider("snapshots"),
+        renewable=config_provider("renewable"),
    input:
-        base_network=RESOURCES + "networks/base.nc",
+        unpack(input_ua_md_availability_matrix),
+        base_network=resources("networks/base.nc"),
        corine=ancient("data/bundle/corine/g250_clc06_V18_5.tif"),
        natura=lambda w: (
-            RESOURCES + "natura.tiff"
-            if config["renewable"][w.technology]["natura"]
+            resources("natura.tiff")
+            if config_provider("renewable", w.technology, "natura")(w)
+            else []
+        ),
+        luisa=lambda w: (
+            "data/LUISA_basemap_020321_50m.tif"
+            if config_provider("renewable", w.technology, "luisa")(w)
            else []
        ),
        gebco=ancient(
            lambda w: (
                "data/bundle/GEBCO_2014_2D.nc"
-                if config["renewable"][w.technology].get("max_depth")
+                if config_provider("renewable", w.technology)(w).get("max_depth")
                else []
            )
        ),
        ship_density=lambda w: (
-            RESOURCES + "shipdensity_raster.tif"
-            if "ship_threshold" in config["renewable"][w.technology].keys()
+            resources("shipdensity_raster.tif")
+            if "ship_threshold" in config_provider("renewable", w.technology)(w).keys()
            else []
        ),
-        country_shapes=RESOURCES + "country_shapes.geojson",
-        offshore_shapes=RESOURCES + "offshore_shapes.geojson",
+        country_shapes=resources("country_shapes.geojson"),
+        offshore_shapes=resources("offshore_shapes.geojson"),
        regions=lambda w: (
-            RESOURCES + "regions_onshore.geojson"
+            resources("regions_onshore.geojson")
            if w.technology in ("onwind", "solar")
-            else RESOURCES + "regions_offshore.geojson"
+            else resources("regions_offshore.geojson")
        ),
        cutout=lambda w: "cutouts/"
        + CDIR
-        + config["renewable"][w.technology]["cutout"]
+        + config_provider("renewable", w.technology, "cutout")(w)
        + ".nc",
    output:
-        profile=RESOURCES + "profile_{technology}.nc",
+        profile=resources("profile_{technology}.nc"),
    log:
-        LOGS + "build_renewable_profile_{technology}.log",
+        logs("build_renewable_profile_{technology}.log"),
    benchmark:
-        BENCHMARKS + "build_renewable_profiles_{technology}"
-    threads: ATLITE_NPROCESSES
+        benchmarks("build_renewable_profiles_{technology}")
+    threads: config["atlite"].get("nprocesses", 4)
    resources:
-        mem_mb=ATLITE_NPROCESSES * 5000,
+        mem_mb=config["atlite"].get("nprocesses", 4) * 5000,
    wildcard_constraints:
        technology="(?!hydro).*",  # Any technology other than hydro
    conda:
@ -262,10 +324,10 @@ rule build_monthly_prices:
        co2_price_raw="data/validation/emission-spot-primary-market-auction-report-2019-data.xls",
        fuel_price_raw="data/validation/energy-price-trends-xlsx-5619002.xlsx",
    output:
-        co2_price=RESOURCES + "co2_price.csv",
-        fuel_price=RESOURCES + "monthly_fuel_price.csv",
+        co2_price=resources("co2_price.csv"),
+        fuel_price=resources("monthly_fuel_price.csv"),
    log:
-        LOGS + "build_monthly_prices.log",
+        logs("build_monthly_prices.log"),
    threads: 1
    resources:
        mem_mb=5000,
@ -277,16 +339,19 @@ rule build_monthly_prices:

 rule build_hydro_profile:
    params:
-        hydro=config["renewable"]["hydro"],
-        countries=config["countries"],
+        hydro=config_provider("renewable", "hydro"),
+        countries=config_provider("countries"),
    input:
-        country_shapes=RESOURCES + "country_shapes.geojson",
+        country_shapes=resources("country_shapes.geojson"),
        eia_hydro_generation="data/eia_hydro_annual_generation.csv",
-        cutout=f"cutouts/" + CDIR + config["renewable"]["hydro"]["cutout"] + ".nc",
+        cutout=lambda w: f"cutouts/"
+        + CDIR
+        + config_provider("renewable", "hydro", "cutout")(w)
+        + ".nc",
    output:
-        RESOURCES + "profile_hydro.nc",
+        resources("profile_hydro.nc"),
    log:
-        LOGS + "build_hydro_profile.log",
+        logs("build_hydro_profile.log"),
    resources:
        mem_mb=5000,
    conda:
@ -295,72 +360,87 @@ rule build_hydro_profile:
        "../scripts/build_hydro_profile.py"


-if config["lines"]["dynamic_line_rating"]["activate"]:
+rule build_line_rating:
+    params:
+        snapshots=config_provider("snapshots"),
+    input:
+        base_network=resources("networks/base.nc"),
+        cutout=lambda w: "cutouts/"
+        + CDIR
+        + config_provider("lines", "dynamic_line_rating", "cutout")(w)
+        + ".nc",
+    output:
+        output=resources("networks/line_rating.nc"),
+    log:
+        logs("build_line_rating.log"),
+    benchmark:
+        benchmarks("build_line_rating")
+    threads: config["atlite"].get("nprocesses", 4)
+    resources:
+        mem_mb=config["atlite"].get("nprocesses", 4) * 1000,
+    conda:
+        "../envs/environment.yaml"
+    script:
+        "../scripts/build_line_rating.py"

-    rule build_line_rating:
-        input:
-            base_network=RESOURCES + "networks/base.nc",
-            cutout="cutouts/"
-            + CDIR
-            + config["lines"]["dynamic_line_rating"]["cutout"]
-            + ".nc",
-        output:
-            output=RESOURCES + "networks/line_rating.nc",
-        log:
-            LOGS + "build_line_rating.log",
-        benchmark:
-            BENCHMARKS + "build_line_rating"
-        threads: ATLITE_NPROCESSES
-        resources:
-            mem_mb=ATLITE_NPROCESSES * 1000,
-        conda:
-            "../envs/environment.yaml"
-        script:
-            "../scripts/build_line_rating.py"
+
+def input_profile_tech(w):
+    return {
+        f"profile_{tech}": resources(f"profile_{tech}.nc")
+        for tech in config_provider("electricity", "renewable_carriers")(w)
+    }
+
+
+def input_conventional(w):
+    return {
+        f"conventional_{carrier}_{attr}": fn
+        for carrier, d in config_provider("conventional", default={None: {}})(w).items()
+        if carrier in config_provider("electricity", "conventional_carriers")(w)
+        for attr, fn in d.items()
+        if str(fn).startswith("data/")
+    }


 rule add_electricity:
    params:
-        length_factor=config["lines"]["length_factor"],
-        scaling_factor=config["load"]["scaling_factor"],
-        countries=config["countries"],
-        renewable=config["renewable"],
-        electricity=config["electricity"],
-        conventional=config["conventional"],
-        costs=config["costs"],
+        length_factor=config_provider("lines", "length_factor"),
+        scaling_factor=config_provider("load", "scaling_factor"),
+        countries=config_provider("countries"),
+        renewable=config_provider("renewable"),
+        electricity=config_provider("electricity"),
+        conventional=config_provider("conventional"),
+        costs=config_provider("costs"),
    input:
-        **{
-            f"profile_{tech}": RESOURCES + f"profile_{tech}.nc"
-            for tech in config["electricity"]["renewable_carriers"]
-        },
-        **{
-            f"conventional_{carrier}_{attr}": fn
-            for carrier, d in config.get("conventional", {None: {}}).items()
-            if carrier in config["electricity"]["conventional_carriers"]
-            for attr, fn in d.items()
-            if str(fn).startswith("data/")
-        },
-        base_network=RESOURCES + "networks/base.nc",
-        line_rating=RESOURCES + "networks/line_rating.nc"
-        if config["lines"]["dynamic_line_rating"]["activate"]
-        else RESOURCES + "networks/base.nc",
-        tech_costs=COSTS,
-        regions=RESOURCES + "regions_onshore.geojson",
-        powerplants=RESOURCES + "powerplants.csv",
+        unpack(input_profile_tech),
+        unpack(input_conventional),
+        base_network=resources("networks/base.nc"),
+        line_rating=lambda w: (
+            resources("networks/line_rating.nc")
+            if config_provider("lines", "dynamic_line_rating", "activate")(w)
+            else resources("networks/base.nc")
+        ),
+        tech_costs=lambda w: resources(
+            f"costs_{config_provider('costs', 'year')(w)}.csv"
+        ),
+        regions=resources("regions_onshore.geojson"),
+        powerplants=resources("powerplants.csv"),
        hydro_capacities=ancient("data/bundle/hydro_capacities.csv"),
        geth_hydro_capacities="data/geth2015_hydro_capacities.csv",
        unit_commitment="data/unit_commitment.csv",
-        fuel_price=RESOURCES + "monthly_fuel_price.csv"
-        if config["conventional"]["dynamic_fuel_price"]
-        else [],
-        load=RESOURCES + "load.csv",
-        nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
+        fuel_price=lambda w: (
+            resources("monthly_fuel_price.csv")
+            if config_provider("conventional", "dynamic_fuel_price")(w)
+            else []
+        ),
+        load=resources("electricity_demand.csv"),
+        nuts3_shapes=resources("nuts3_shapes.geojson"),
+        ua_md_gdp="data/GDP_PPP_30arcsec_v3_mapped_default.csv",
    output:
-        RESOURCES + "networks/elec.nc",
+        resources("networks/elec.nc"),
    log:
-        LOGS + "add_electricity.log",
+        logs("add_electricity.log"),
    benchmark:
-        BENCHMARKS + "add_electricity"
+        benchmarks("add_electricity")
    threads: 1
    resources:
        mem_mb=10000,
@ -372,29 +452,33 @@ rule add_electricity:

 rule simplify_network:
    params:
-        simplify_network=config["clustering"]["simplify_network"],
-        aggregation_strategies=config["clustering"].get("aggregation_strategies", {}),
-        focus_weights=config.get("focus_weights", None),
-        renewable_carriers=config["electricity"]["renewable_carriers"],
-        max_hours=config["electricity"]["max_hours"],
-        length_factor=config["lines"]["length_factor"],
-        p_max_pu=config["links"].get("p_max_pu", 1.0),
-        costs=config["costs"],
+        simplify_network=config_provider("clustering", "simplify_network"),
+        aggregation_strategies=config_provider(
+            "clustering", "aggregation_strategies", default={}
+        ),
+        focus_weights=config_provider("clustering", "focus_weights", default=None),
+        renewable_carriers=config_provider("electricity", "renewable_carriers"),
+        max_hours=config_provider("electricity", "max_hours"),
+        length_factor=config_provider("lines", "length_factor"),
+        p_max_pu=config_provider("links", "p_max_pu", default=1.0),
+        costs=config_provider("costs"),
    input:
-        network=RESOURCES + "networks/elec.nc",
-        tech_costs=COSTS,
-        regions_onshore=RESOURCES + "regions_onshore.geojson",
-        regions_offshore=RESOURCES + "regions_offshore.geojson",
+        network=resources("networks/elec.nc"),
+        tech_costs=lambda w: resources(
+            f"costs_{config_provider('costs', 'year')(w)}.csv"
+        ),
+        regions_onshore=resources("regions_onshore.geojson"),
+        regions_offshore=resources("regions_offshore.geojson"),
    output:
-        network=RESOURCES + "networks/elec_s{simpl}.nc",
-        regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson",
-        regions_offshore=RESOURCES + "regions_offshore_elec_s{simpl}.geojson",
-        busmap=RESOURCES + "busmap_elec_s{simpl}.csv",
-        connection_costs=RESOURCES + "connection_costs_s{simpl}.csv",
+        network=resources("networks/elec_s{simpl}.nc"),
+        regions_onshore=resources("regions_onshore_elec_s{simpl}.geojson"),
+        regions_offshore=resources("regions_offshore_elec_s{simpl}.geojson"),
+        busmap=resources("busmap_elec_s{simpl}.csv"),
+        connection_costs=resources("connection_costs_s{simpl}.csv"),
    log:
-        LOGS + "simplify_network/elec_s{simpl}.log",
+        logs("simplify_network/elec_s{simpl}.log"),
    benchmark:
-        BENCHMARKS + "simplify_network/elec_s{simpl}"
+        benchmarks("simplify_network/elec_s{simpl}")
    threads: 1
    resources:
        mem_mb=12000,
@ -406,36 +490,42 @@ rule simplify_network:

 rule cluster_network:
    params:
-        cluster_network=config["clustering"]["cluster_network"],
-        aggregation_strategies=config["clustering"].get("aggregation_strategies", {}),
-        custom_busmap=config["enable"].get("custom_busmap", False),
-        focus_weights=config.get("focus_weights", None),
-        renewable_carriers=config["electricity"]["renewable_carriers"],
-        conventional_carriers=config["electricity"].get("conventional_carriers", []),
-        max_hours=config["electricity"]["max_hours"],
-        length_factor=config["lines"]["length_factor"],
-        costs=config["costs"],
+        cluster_network=config_provider("clustering", "cluster_network"),
+        aggregation_strategies=config_provider(
+            "clustering", "aggregation_strategies", default={}
+        ),
+        custom_busmap=config_provider("enable", "custom_busmap", default=False),
+        focus_weights=config_provider("clustering", "focus_weights", default=None),
+        renewable_carriers=config_provider("electricity", "renewable_carriers"),
+        conventional_carriers=config_provider(
+            "electricity", "conventional_carriers", default=[]
+        ),
+        max_hours=config_provider("electricity", "max_hours"),
+        length_factor=config_provider("lines", "length_factor"),
+        costs=config_provider("costs"),
    input:
-        network=RESOURCES + "networks/elec_s{simpl}.nc",
-        regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson",
-        regions_offshore=RESOURCES + "regions_offshore_elec_s{simpl}.geojson",
-        busmap=ancient(RESOURCES + "busmap_elec_s{simpl}.csv"),
-        custom_busmap=(
+        network=resources("networks/elec_s{simpl}.nc"),
+        regions_onshore=resources("regions_onshore_elec_s{simpl}.geojson"),
+        regions_offshore=resources("regions_offshore_elec_s{simpl}.geojson"),
+        busmap=ancient(resources("busmap_elec_s{simpl}.csv")),
+        custom_busmap=lambda w: (
            "data/custom_busmap_elec_s{simpl}_{clusters}.csv"
-            if config["enable"].get("custom_busmap", False)
+            if config_provider("enable", "custom_busmap", default=False)(w)
            else []
        ),
-        tech_costs=COSTS,
+        tech_costs=lambda w: resources(
+            f"costs_{config_provider('costs', 'year')(w)}.csv"
+        ),
    output:
-        network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc",
-        regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
-        regions_offshore=RESOURCES + "regions_offshore_elec_s{simpl}_{clusters}.geojson",
-        busmap=RESOURCES + "busmap_elec_s{simpl}_{clusters}.csv",
-        linemap=RESOURCES + "linemap_elec_s{simpl}_{clusters}.csv",
+        network=resources("networks/elec_s{simpl}_{clusters}.nc"),
+        regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
+        regions_offshore=resources("regions_offshore_elec_s{simpl}_{clusters}.geojson"),
+        busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"),
+        linemap=resources("linemap_elec_s{simpl}_{clusters}.csv"),
    log:
-        LOGS + "cluster_network/elec_s{simpl}_{clusters}.log",
+        logs("cluster_network/elec_s{simpl}_{clusters}.log"),
    benchmark:
-        BENCHMARKS + "cluster_network/elec_s{simpl}_{clusters}"
+        benchmarks("cluster_network/elec_s{simpl}_{clusters}")
    threads: 1
    resources:
        mem_mb=10000,
@ -447,18 +537,20 @@ rule cluster_network:

 rule add_extra_components:
    params:
-        extendable_carriers=config["electricity"]["extendable_carriers"],
-        max_hours=config["electricity"]["max_hours"],
-        costs=config["costs"],
+        extendable_carriers=config_provider("electricity", "extendable_carriers"),
+        max_hours=config_provider("electricity", "max_hours"),
+        costs=config_provider("costs"),
    input:
-        network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc",
-        tech_costs=COSTS,
+        network=resources("networks/elec_s{simpl}_{clusters}.nc"),
+        tech_costs=lambda w: resources(
+            f"costs_{config_provider('costs', 'year')(w)}.csv"
+        ),
    output:
-        RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc",
+        resources("networks/elec_s{simpl}_{clusters}_ec.nc"),
    log:
-        LOGS + "add_extra_components/elec_s{simpl}_{clusters}.log",
+        logs("add_extra_components/elec_s{simpl}_{clusters}.log"),
    benchmark:
-        BENCHMARKS + "add_extra_components/elec_s{simpl}_{clusters}_ec"
+        benchmarks("add_extra_components/elec_s{simpl}_{clusters}_ec")
    threads: 1
    resources:
        mem_mb=4000,
@ -470,23 +562,30 @@ rule add_extra_components:

 rule prepare_network:
    params:
-        links=config["links"],
-        lines=config["lines"],
-        co2base=config["electricity"]["co2base"],
-        co2limit=config["electricity"]["co2limit"],
-        gaslimit=config["electricity"].get("gaslimit"),
-        max_hours=config["electricity"]["max_hours"],
-        costs=config["costs"],
+        time_resolution=config_provider("clustering", "temporal", "resolution_elec"),
+        links=config_provider("links"),
+        lines=config_provider("lines"),
+        co2base=config_provider("electricity", "co2base"),
+        co2limit_enable=config_provider("electricity", "co2limit_enable", default=False),
+        co2limit=config_provider("electricity", "co2limit"),
+        gaslimit_enable=config_provider("electricity", "gaslimit_enable", default=False),
+        gaslimit=config_provider("electricity", "gaslimit"),
+        max_hours=config_provider("electricity", "max_hours"),
+        costs=config_provider("costs"),
+        adjustments=config_provider("adjustments", "electricity"),
+        autarky=config_provider("electricity", "autarky", default={}),
    input:
-        RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc",
-        tech_costs=COSTS,
-        co2_price=lambda w: RESOURCES + "co2_price.csv" if "Ept" in w.opts else [],
+        resources("networks/elec_s{simpl}_{clusters}_ec.nc"),
+        tech_costs=lambda w: resources(
+            f"costs_{config_provider('costs', 'year')(w)}.csv"
+        ),
+        co2_price=lambda w: resources("co2_price.csv") if "Ept" in w.opts else [],
    output:
-        RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
+        resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"),
    log:
-        LOGS + "prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.log",
+        logs("prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.log"),
    benchmark:
-        (BENCHMARKS + "prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}")
+        (benchmarks("prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}"))
    threads: 1
    resources:
        mem_mb=4000,
--- a/rules/build_sector.smk
+++ b/rules/build_sector.smk
--- a/rules/collect.smk
+++ b/rules/collect.smk
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2023-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT

@ -11,26 +11,32 @@ localrules:
    prepare_sector_networks,
    solve_elec_networks,
    solve_sector_networks,
-    plot_networks,


 rule cluster_networks:
    input:
-        expand(RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", **config["scenario"]),
+        expand(
+            resources("networks/elec_s{simpl}_{clusters}.nc"),
+            **config["scenario"],
+            run=config["run"]["name"],
+        ),


 rule extra_components_networks:
    input:
        expand(
-            RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", **config["scenario"]
+            resources("networks/elec_s{simpl}_{clusters}_ec.nc"),
+            **config["scenario"],
+            run=config["run"]["name"],
        ),


 rule prepare_elec_networks:
    input:
        expand(
-            RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
-            **config["scenario"]
+            resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"),
+            **config["scenario"],
+            run=config["run"]["name"],
        ),


@ -39,7 +45,8 @@ rule prepare_sector_networks:
        expand(
            RESULTS
            + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-            **config["scenario"]
+            **config["scenario"],
+            run=config["run"]["name"],
        ),


@ -47,7 +54,8 @@ rule solve_elec_networks:
    input:
        expand(
            RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
-            **config["scenario"]
+            **config["scenario"],
+            run=config["run"]["name"],
        ),


@ -56,25 +64,18 @@ rule solve_sector_networks:
        expand(
            RESULTS
            + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-            **config["scenario"]
+            **config["scenario"],
+            run=config["run"]["name"],
        ),


 rule solve_sector_networks_perfect:
-    input:
-        expand(
-            RESULTS
-            + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
-            **config["scenario"]
-        ),
-
-
-rule plot_networks:
    input:
        expand(
            RESULTS
            + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf",
-            **config["scenario"]
+            **config["scenario"],
+            run=config["run"]["name"],
        ),


@ -83,11 +84,13 @@ rule validate_elec_networks:
        expand(
            RESULTS
            + "figures/.statistics_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}",
-            **config["scenario"]
+            **config["scenario"],
+            run=config["run"]["name"],
        ),
        expand(
            RESULTS
            + "figures/.validation_{kind}_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}",
            **config["scenario"],
-            kind=["production", "prices", "cross_border"]
+            run=config["run"]["name"],
+            kind=["production", "prices", "cross_border"],
        ),
--- a/rules/common.smk
+++ b/rules/common.smk
@ -1,7 +1,89 @@
-# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2023-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT

+import copy
+from functools import partial, lru_cache
+
+import os, sys, glob
+
+path = workflow.source_path("../scripts/_helpers.py")
+sys.path.insert(0, os.path.dirname(path))
+
+from _helpers import validate_checksum, update_config_from_wildcards
+from snakemake.utils import update_config
+
+
+def get_config(config, keys, default=None):
+    """Retrieve a nested value from a dictionary using a tuple of keys."""
+    value = config
+    for key in keys:
+        if isinstance(value, list):
+            value = value[key]
+        else:
+            value = value.get(key, default)
+        if value == default:
+            return default
+    return value
+
+
+def merge_configs(base_config, scenario_config):
+    """Merge base config with a specific scenario without modifying the original."""
+    merged = copy.deepcopy(base_config)
+    update_config(merged, scenario_config)
+    return merged
+
+
+@lru_cache
+def scenario_config(scenario_name):
+    """Retrieve a scenario config based on the overrides from the scenario file."""
+    return merge_configs(config, scenarios[scenario_name])
+
+
+def static_getter(wildcards, keys, default):
+    """Getter function for static config values."""
+    config_with_wildcards = update_config_from_wildcards(
+        config, wildcards, inplace=False
+    )
+    return get_config(config_with_wildcards, keys, default)
+
+
+def dynamic_getter(wildcards, keys, default):
+    """Getter function for dynamic config values based on scenario."""
+    if "run" not in wildcards.keys():
+        return get_config(config, keys, default)
+    scenario_name = wildcards.run
+    if scenario_name not in scenarios:
+        raise ValueError(
+            f"Scenario {scenario_name} not found in file {config['run']['scenario']['file']}."
+        )
+    config_with_scenario = scenario_config(scenario_name)
+    config_with_wildcards = update_config_from_wildcards(
+        config_with_scenario, wildcards, inplace=False
+    )
+    return get_config(config_with_wildcards, keys, default)
+
+
+def config_provider(*keys, default=None):
+    """Dynamically provide config values based on 'run' -> 'name'.
+
+    Usage in Snakemake rules would look something like:
+    params:
+        my_param=config_provider("key1", "key2", default="some_default_value")
+    """
+    # Using functools.partial to freeze certain arguments in our getter functions.
+    if config["run"].get("scenarios", {}).get("enable", False):
+        return partial(dynamic_getter, keys=keys, default=default)
+    else:
+        return partial(static_getter, keys=keys, default=default)
+
+
+def solver_threads(w):
+    solver_options = config_provider("solving", "solver_options")(w)
+    option_set = config_provider("solving", "solver", "options")(w)
+    threads = solver_options[option_set].get("threads", 4)
+    return threads
+

 def memory(w):
    factor = 3.0
@ -23,6 +105,15 @@ def memory(w):
        return int(factor * (10000 + 195 * int(w.clusters)))


+def input_custom_extra_functionality(w):
+    path = config_provider(
+        "solving", "options", "custom_extra_functionality", default=False
+    )(w)
+    if path:
+        return os.path.join(os.path.dirname(workflow.snakefile), path)
+    return []
+
+
 # Check if the workflow has access to the internet by trying to access the HEAD of specified url
 def has_internet_access(url="www.zenodo.org") -> bool:
    import http.client as http_client
@ -39,16 +130,11 @@ def has_internet_access(url="www.zenodo.org") -> bool:
        conn.close()


-def input_eurostat(w):
-    # 2016 includes BA, 2017 does not
-    report_year = config["energy"]["eurostat_report_year"]
-    return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition"
-
-
-def solved_previous_horizon(wildcards):
-    planning_horizons = config["scenario"]["planning_horizons"]
-    i = planning_horizons.index(int(wildcards.planning_horizons))
+def solved_previous_horizon(w):
+    planning_horizons = config_provider("scenario", "planning_horizons")(w)
+    i = planning_horizons.index(int(w.planning_horizons))
    planning_horizon_p = str(planning_horizons[i - 1])
+
    return (
        RESULTS
        + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_"
--- a/rules/postprocess.smk
+++ b/rules/postprocess.smk
@ -1,69 +1,140 @@
-# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2023-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT


 localrules:
    copy_config,
-    copy_conda_env,


 if config["foresight"] != "perfect":

-    rule plot_network:
+    rule plot_power_network_clustered:
        params:
-            foresight=config["foresight"],
-            plotting=config["plotting"],
+            plotting=config_provider("plotting"),
+        input:
+            network=resources("networks/elec_s{simpl}_{clusters}.nc"),
+            regions_onshore=resources(
+                "regions_onshore_elec_s{simpl}_{clusters}.geojson"
+            ),
+        output:
+            map=resources("maps/power-network-s{simpl}-{clusters}.pdf"),
+        threads: 1
+        resources:
+            mem_mb=4000,
+        benchmark:
+            benchmarks("plot_power_network_clustered/elec_s{simpl}_{clusters}")
+        conda:
+            "../envs/environment.yaml"
+        script:
+            "../scripts/plot_power_network_clustered.py"
+
+    rule plot_power_network:
+        params:
+            plotting=config_provider("plotting"),
        input:
            network=RESULTS
            + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-            regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
+            regions=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
        output:
            map=RESULTS
            + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf",
-            today=RESULTS
-            + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}-today.pdf",
        threads: 2
        resources:
            mem_mb=10000,
+        log:
+            RESULTS
+            + "logs/plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
        benchmark:
            (
-                BENCHMARKS
-                + "plot_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
+                RESULTS
+                + "benchmarksplot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
            )
        conda:
            "../envs/environment.yaml"
        script:
-            "../scripts/plot_network.py"
+            "../scripts/plot_power_network.py"
+
+    rule plot_hydrogen_network:
+        params:
+            plotting=config_provider("plotting"),
+            foresight=config_provider("foresight"),
+        input:
+            network=RESULTS
+            + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
+            regions=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
+        output:
+            map=RESULTS
+            + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf",
+        threads: 2
+        resources:
+            mem_mb=10000,
+        log:
+            RESULTS
+            + "logs/plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
+        benchmark:
+            (
+                RESULTS
+                + "benchmarks/plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
+            )
+        conda:
+            "../envs/environment.yaml"
+        script:
+            "../scripts/plot_hydrogen_network.py"
+
+    rule plot_gas_network:
+        params:
+            plotting=config_provider("plotting"),
+        input:
+            network=RESULTS
+            + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
+            regions=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
+        output:
+            map=RESULTS
+            + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-ch4_network_{planning_horizons}.pdf",
+        threads: 2
+        resources:
+            mem_mb=10000,
+        log:
+            RESULTS
+            + "logs/plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
+        benchmark:
+            (
+                RESULTS
+                + "benchmarks/plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
+            )
+        conda:
+            "../envs/environment.yaml"
+        script:
+            "../scripts/plot_gas_network.py"


 if config["foresight"] == "perfect":

-    rule plot_network:
+    def output_map_year(w):
+        return {
+            f"map_{year}": RESULTS
+            + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_"
+            + f"{year}.pdf"
+            for year in config_provider("scenario", "planning_horizons")(w)
+        }
+
+    rule plot_power_network_perfect:
        params:
-            foresight=config["foresight"],
-            plotting=config["plotting"],
+            plotting=config_provider("plotting"),
        input:
            network=RESULTS
            + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
-            regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
+            regions=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
        output:
-            **{
-                f"map_{year}": RESULTS
-                + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_"
-                + f"{year}.pdf"
-                for year in config["scenario"]["planning_horizons"]
-            },
+            unpack(output_map_year),
        threads: 2
        resources:
            mem_mb=10000,
-        benchmark:
-            BENCHMARKS
-            +"postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_benchmark"
        conda:
            "../envs/environment.yaml"
        script:
-            "../scripts/plot_network.py"
+            "../scripts/plot_power_network_perfect.py"


 rule copy_config:
@ -74,8 +145,6 @@ rule copy_config:
    threads: 1
    resources:
        mem_mb=1000,
-    benchmark:
-        BENCHMARKS + "copy_config"
    conda:
        "../envs/environment.yaml"
    script:
@ -84,24 +153,57 @@ rule copy_config:

 rule make_summary:
    params:
-        foresight=config["foresight"],
-        costs=config["costs"],
-        snapshots=config["snapshots"],
-        scenario=config["scenario"],
+        foresight=config_provider("foresight"),
+        costs=config_provider("costs"),
+        snapshots=config_provider("snapshots"),
+        scenario=config_provider("scenario"),
        RDIR=RDIR,
    input:
        networks=expand(
            RESULTS
            + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-            **config["scenario"]
+            **config["scenario"],
+            allow_missing=True,
        ),
-        costs="data/costs_{}.csv".format(config["costs"]["year"])
-        if config["foresight"] == "overnight"
-        else "data/costs_{}.csv".format(config["scenario"]["planning_horizons"][0]),
-        plots=expand(
+        costs=lambda w: (
+            resources("costs_{}.csv".format(config_provider("costs", "year")(w)))
+            if config_provider("foresight")(w) == "overnight"
+            else resources(
+                "costs_{}.csv".format(
+                    config_provider("scenario", "planning_horizons", 0)(w)
+                )
+            )
+        ),
+        ac_plot=expand(
+            resources("maps/power-network-s{simpl}-{clusters}.pdf"),
+            **config["scenario"],
+            allow_missing=True,
+        ),
+        costs_plot=expand(
            RESULTS
            + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf",
-            **config["scenario"]
+            **config["scenario"],
+            allow_missing=True,
+        ),
+        h2_plot=lambda w: expand(
+            (
+                RESULTS
+                + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf"
+                if config_provider("sector", "H2_network")(w)
+                else []
+            ),
+            **config["scenario"],
+            allow_missing=True,
+        ),
+        ch4_plot=lambda w: expand(
+            (
+                RESULTS
+                + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-ch4_network_{planning_horizons}.pdf"
+                if config_provider("sector", "gas_network")(w)
+                else []
+            ),
+            **config["scenario"],
+            allow_missing=True,
        ),
    output:
        nodal_costs=RESULTS + "csvs/nodal_costs.csv",
@ -123,9 +225,7 @@ rule make_summary:
    resources:
        mem_mb=10000,
    log:
-        LOGS + "make_summary.log",
-    benchmark:
-        BENCHMARKS + "make_summary"
+        RESULTS + "logs/make_summary.log",
    conda:
        "../envs/environment.yaml"
    script:
@ -134,18 +234,19 @@ rule make_summary:

 rule plot_summary:
    params:
-        countries=config["countries"],
-        planning_horizons=config["scenario"]["planning_horizons"],
-        sector_opts=config["scenario"]["sector_opts"],
-        emissions_scope=config["energy"]["emissions"],
-        eurostat_report_year=config["energy"]["eurostat_report_year"],
-        plotting=config["plotting"],
+        countries=config_provider("countries"),
+        planning_horizons=config_provider("scenario", "planning_horizons"),
+        emissions_scope=config_provider("energy", "emissions"),
+        plotting=config_provider("plotting"),
+        foresight=config_provider("foresight"),
+        co2_budget=config_provider("co2_budget"),
+        sector=config_provider("sector"),
        RDIR=RDIR,
    input:
        costs=RESULTS + "csvs/costs.csv",
        energy=RESULTS + "csvs/energy.csv",
        balances=RESULTS + "csvs/supply_energy.csv",
-        eurostat=input_eurostat,
+        eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition",
        co2="data/bundle-sector/eea/UNFCCC_v23.csv",
    output:
        costs=RESULTS + "graphs/costs.pdf",
@ -155,9 +256,7 @@ rule plot_summary:
    resources:
        mem_mb=10000,
    log:
-        LOGS + "plot_summary.log",
-    benchmark:
-        BENCHMARKS + "plot_summary"
+        RESULTS + "logs/plot_summary.log",
    conda:
        "../envs/environment.yaml"
    script:
@ -179,7 +278,7 @@ STATISTICS_BARPLOTS = [

 rule plot_elec_statistics:
    params:
-        plotting=config["plotting"],
+        plotting=config_provider("plotting"),
        barplots=STATISTICS_BARPLOTS,
    input:
        network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
--- a/rules/retrieve.smk
+++ b/rules/retrieve.smk
@ -1,7 +1,10 @@
-# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2023-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT

+import requests
+from datetime import datetime, timedelta
+
 if config["enable"].get("retrieve", "auto") == "auto":
    config["enable"]["retrieve"] = has_internet_access()

@ -29,12 +32,12 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle",
        output:
            protected(expand("data/bundle/{file}", file=datafiles)),
        log:
-            LOGS + "retrieve_databundle.log",
+            "logs/retrieve_databundle.log",
        resources:
            mem_mb=1000,
        retries: 2
        conda:
-            "../envs/environment.yaml"
+            "../envs/retrieve.yaml"
        script:
            "../scripts/retrieve_databundle.py"

@ -47,12 +50,12 @@ if config["enable"].get("retrieve_irena"):
            onwind="data/existing_infrastructure/onwind_capacity_IRENA.csv",
            solar="data/existing_infrastructure/solar_capacity_IRENA.csv",
        log:
-            LOGS + "retrieve_irena.log",
+            logs("retrieve_irena.log"),
        resources:
            mem_mb=1000,
        retries: 2
        conda:
-            "../envs/environment.yaml"
+            "../envs/retrieve.yaml"
        script:
            "../scripts/retrieve_irena.py"

@ -61,9 +64,8 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True

    rule retrieve_cutout:
        input:
-            HTTP.remote(
-                "zenodo.org/record/6382570/files/{cutout}.nc",
-                static=True,
+            storage(
+                "https://zenodo.org/record/6382570/files/{cutout}.nc",
            ),
        output:
            protected("cutouts/" + CDIR + "{cutout}.nc"),
@ -74,28 +76,25 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True
        retries: 2
        run:
            move(input[0], output[0])
+            validate_checksum(output[0], input[0])


 if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", True):

    rule retrieve_cost_data:
-        input:
-            HTTP.remote(
-                "raw.githubusercontent.com/PyPSA/technology-data/{}/outputs/".format(
-                    config["costs"]["version"]
-                )
-                + "costs_{year}.csv",
-                keep_local=True,
-            ),
+        params:
+            version=config_provider("costs", "version"),
        output:
-            "data/costs_{year}.csv",
+            resources("costs_{year}.csv"),
        log:
-            LOGS + "retrieve_cost_data_{year}.log",
+            logs("retrieve_cost_data_{year}.log"),
        resources:
            mem_mb=1000,
        retries: 2
-        run:
-            move(input[0], output[0])
+        conda:
+            "../envs/retrieve.yaml"
+        script:
+            "../scripts/retrieve_cost_data.py"


 if config["enable"]["retrieve"] and config["enable"].get(
@ -104,20 +103,20 @@ if config["enable"]["retrieve"] and config["enable"].get(

    rule retrieve_natura_raster:
        input:
-            HTTP.remote(
-                "zenodo.org/record/4706686/files/natura.tiff",
+            storage(
+                "https://zenodo.org/record/4706686/files/natura.tiff",
                keep_local=True,
-                static=True,
            ),
        output:
-            protected(RESOURCES + "natura.tiff"),
+            resources("natura.tiff"),
        log:
-            LOGS + "retrieve_natura_raster.log",
+            logs("retrieve_natura_raster.log"),
        resources:
            mem_mb=5000,
        retries: 2
        run:
            move(input[0], output[0])
+            validate_checksum(output[0], input[0])


 if config["enable"]["retrieve"] and config["enable"].get(
@ -134,49 +133,45 @@ if config["enable"]["retrieve"] and config["enable"].get(
        "h2_salt_caverns_GWh_per_sqkm.geojson",
    ]

-    datafolders = [
-        protected(
-            directory("data/bundle-sector/eurostat-energy_balances-june_2016_edition")
-        ),
-        protected(
-            directory("data/bundle-sector/eurostat-energy_balances-may_2018_edition")
-        ),
-        protected(directory("data/bundle-sector/jrc-idees-2015")),
-    ]
-
    rule retrieve_sector_databundle:
        output:
            protected(expand("data/bundle-sector/{files}", files=datafiles)),
-            *datafolders,
+            protected(directory("data/bundle-sector/jrc-idees-2015")),
        log:
-            LOGS + "retrieve_sector_databundle.log",
+            "logs/retrieve_sector_databundle.log",
        retries: 2
        conda:
-            "../envs/environment.yaml"
+            "../envs/retrieve.yaml"
        script:
            "../scripts/retrieve_sector_databundle.py"

+    rule retrieve_eurostat_data:
+        output:
+            directory("data/eurostat/eurostat-energy_balances-april_2023_edition"),
+        log:
+            "logs/retrieve_eurostat_data.log",
+        retries: 2
+        script:
+            "../scripts/retrieve_eurostat_data.py"

-if config["enable"]["retrieve"] and (
-    config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]
-):
+
+if config["enable"]["retrieve"]:
    datafiles = [
        "IGGIELGN_LNGs.geojson",
        "IGGIELGN_BorderPoints.geojson",
        "IGGIELGN_Productions.geojson",
+        "IGGIELGN_Storages.geojson",
        "IGGIELGN_PipeSegments.geojson",
    ]

    rule retrieve_gas_infrastructure_data:
        output:
-            protected(
-                expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles)
-            ),
+            expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles),
        log:
-            LOGS + "retrieve_gas_infrastructure_data.log",
+            "logs/retrieve_gas_infrastructure_data.log",
        retries: 2
        conda:
-            "../envs/environment.yaml"
+            "../envs/retrieve.yaml"
        script:
            "../scripts/retrieve_gas_infrastructure_data.py"

@ -184,60 +179,161 @@ if config["enable"]["retrieve"] and (
 if config["enable"]["retrieve"]:

    rule retrieve_electricity_demand:
-        input:
-            HTTP.remote(
-                "data.open-power-system-data.org/time_series/{version}/time_series_60min_singleindex.csv".format(
-                version="2019-06-05"
-                    if config["snapshots"]["end"] < "2019"
-                    else "2020-10-06"
-                ),
-                keep_local=True,
-                static=True,
-            ),
+        params:
+            versions=["2019-06-05", "2020-10-06"],
        output:
-            "data/load_raw.csv",
+            "data/electricity_demand_raw.csv",
        log:
-            LOGS + "retrieve_electricity_demand.log",
+            "logs/retrieve_electricity_demand.log",
        resources:
            mem_mb=5000,
        retries: 2
-        run:
-            move(input[0], output[0])
+        conda:
+            "../envs/retrieve.yaml"
+        script:
+            "../scripts/retrieve_electricity_demand.py"


 if config["enable"]["retrieve"]:

    rule retrieve_ship_raster:
        input:
-            HTTP.remote(
+            storage(
                "https://zenodo.org/record/6953563/files/shipdensity_global.zip",
                keep_local=True,
-                static=True,
            ),
        output:
            protected("data/shipdensity_global.zip"),
        log:
-            LOGS + "retrieve_ship_raster.log",
+            "logs/retrieve_ship_raster.log",
        resources:
            mem_mb=5000,
        retries: 2
        run:
            move(input[0], output[0])
+            validate_checksum(output[0], input[0])
+
+
+if config["enable"]["retrieve"]:
+
+    # Downloading Copernicus Global Land Cover for land cover and land use:
+    # Website: https://land.copernicus.eu/global/products/lc
+    rule download_copernicus_land_cover:
+        input:
+            storage(
+                "https://zenodo.org/record/3939050/files/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif",
+            ),
+        output:
+            "data/Copernicus_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif",
+        run:
+            move(input[0], output[0])
+            validate_checksum(output[0], input[0])
+
+
+if config["enable"]["retrieve"]:
+
+    # Downloading LUISA Base Map for land cover and land use:
+    # Website: https://ec.europa.eu/jrc/en/luisa
+    rule retrieve_luisa_land_cover:
+        input:
+            storage(
+                "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/LUISA/EUROPE/Basemaps/LandUse/2018/LATEST/LUISA_basemap_020321_50m.tif",
+            ),
+        output:
+            "data/LUISA_basemap_020321_50m.tif",
+        run:
+            move(input[0], output[0])
+
+
+if config["enable"]["retrieve"]:
+    # Some logic to find the correct file URL
+    # Sometimes files are released delayed or ahead of schedule, check which file is currently available
+
+    def check_file_exists(url):
+        response = requests.head(url)
+        return response.status_code == 200
+
+    # Basic pattern where WDPA files can be found
+    url_pattern = (
+        "https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_{bYYYY}_Public_shp.zip"
+    )
+
+    # 3-letter month + 4 digit year for current/previous/next month to test
+    current_monthyear = datetime.now().strftime("%b%Y")
+    prev_monthyear = (datetime.now() - timedelta(30)).strftime("%b%Y")
+    next_monthyear = (datetime.now() + timedelta(30)).strftime("%b%Y")
+
+    # Test prioritised: current month -> previous -> next
+    for bYYYY in [current_monthyear, prev_monthyear, next_monthyear]:
+        if check_file_exists(url := url_pattern.format(bYYYY=bYYYY)):
+            break
+        else:
+            # If None of the three URLs are working
+            url = False
+
+    assert (
+        url
+    ), f"No WDPA files found at {url_pattern} for bY='{current_monthyear}, {prev_monthyear}, or {next_monthyear}'"
+
+    # Downloading protected area database from WDPA
+    # extract the main zip and then merge the contained 3 zipped shapefiles
+    # Website: https://www.protectedplanet.net/en/thematic-areas/wdpa
+    rule download_wdpa:
+        input:
+            storage(url, keep_local=True),
+        params:
+            zip="data/WDPA_shp.zip",
+            folder=directory("data/WDPA"),
+        output:
+            gpkg=protected("data/WDPA.gpkg"),
+        run:
+            shell("cp {input} {params.zip}")
+            shell("unzip -o {params.zip} -d {params.folder}")
+            for i in range(3):
+                # vsizip is special driver for directly working with zipped shapefiles in ogr2ogr
+                layer_path = (
+                    f"/vsizip/{params.folder}/WDPA_{bYYYY}_Public_shp_{i}.zip"
+                )
+                print(f"Adding layer {i + 1} of 3 to combined output file.")
+                shell("ogr2ogr -f gpkg -update -append {output.gpkg} {layer_path}")
+
+    rule download_wdpa_marine:
+        # Downloading Marine protected area database from WDPA
+        # extract the main zip and then merge the contained 3 zipped shapefiles
+        # Website: https://www.protectedplanet.net/en/thematic-areas/marine-protected-areas
+        input:
+            storage(
+                f"https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_WDOECM_{bYYYY}_Public_marine_shp.zip",
+                keep_local=True,
+            ),
+        params:
+            zip="data/WDPA_WDOECM_marine.zip",
+            folder=directory("data/WDPA_WDOECM_marine"),
+        output:
+            gpkg=protected("data/WDPA_WDOECM_marine.gpkg"),
+        run:
+            shell("cp {input} {params.zip}")
+            shell("unzip -o {params.zip} -d {params.folder}")
+            for i in range(3):
+                # vsizip is special driver for directly working with zipped shapefiles in ogr2ogr
+                layer_path = f"/vsizip/{params.folder}/WDPA_WDOECM_{bYYYY}_Public_marine_shp_{i}.zip"
+                print(f"Adding layer {i + 1} of 3 to combined output file.")
+                shell("ogr2ogr -f gpkg -update -append {output.gpkg} {layer_path}")
+


 if config["enable"]["retrieve"]:

    rule retrieve_monthly_co2_prices:
        input:
-            HTTP.remote(
+            storage(
                "https://www.eex.com/fileadmin/EEX/Downloads/EUA_Emission_Spot_Primary_Market_Auction_Report/Archive_Reports/emission-spot-primary-market-auction-report-2019-data.xls",
                keep_local=True,
-                static=True,
            ),
        output:
            "data/validation/emission-spot-primary-market-auction-report-2019-data.xls",
        log:
-            LOGS + "retrieve_monthly_co2_prices.log",
+            "logs/retrieve_monthly_co2_prices.log",
        resources:
            mem_mb=5000,
        retries: 2
@ -251,11 +347,11 @@ if config["enable"]["retrieve"]:
        output:
            "data/validation/energy-price-trends-xlsx-5619002.xlsx",
        log:
-            LOGS + "retrieve_monthly_fuel_prices.log",
+            "logs/retrieve_monthly_fuel_prices.log",
        resources:
            mem_mb=5000,
        retries: 2
        conda:
-            "../envs/environment.yaml"
+            "../envs/retrieve.yaml"
        script:
            "../scripts/retrieve_monthly_fuel_prices.py"
--- a/rules/solve_electricity.smk
+++ b/rules/solve_electricity.smk
@ -1,33 +1,35 @@
-# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2023-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT


 rule solve_network:
    params:
-        solving=config["solving"],
-        foresight=config["foresight"],
-        planning_horizons=config["scenario"]["planning_horizons"],
-        co2_sequestration_potential=config["sector"].get(
-            "co2_sequestration_potential", 200
+        solving=config_provider("solving"),
+        foresight=config_provider("foresight"),
+        planning_horizons=config_provider("scenario", "planning_horizons"),
+        co2_sequestration_potential=config_provider(
+            "sector", "co2_sequestration_potential", default=200
        ),
+        custom_extra_functionality=input_custom_extra_functionality,
    input:
-        network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
+        network=resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"),
        config=RESULTS + "config.yaml",
    output:
        network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
    log:
        solver=normpath(
-            LOGS + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"
+            RESULTS
+            + "logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"
        ),
-        python=LOGS
-        + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log",
+        python=RESULTS
+        + "logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log",
    benchmark:
-        BENCHMARKS + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}"
-    threads: 4
+        RESULTS + "benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}"
+    threads: solver_threads
    resources:
        mem_mb=memory,
-        walltime=config["solving"].get("walltime", "12:00:00"),
+        runtime=config_provider("solving", "runtime", default="6h"),
    shadow:
        "minimal"
    conda:
@ -38,27 +40,27 @@ rule solve_network:

 rule solve_operations_network:
    params:
-        options=config["solving"]["options"],
+        options=config_provider("solving", "options"),
    input:
        network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
    output:
        network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc",
    log:
        solver=normpath(
-            LOGS
-            + "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"
+            RESULTS
+            + "logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"
        ),
-        python=LOGS
-        + "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log",
+        python=RESULTS
+        + "logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log",
    benchmark:
        (
-            BENCHMARKS
-            + "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}"
+            RESULTS
+            + "benchmarks/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}"
        )
    threads: 4
    resources:
        mem_mb=(lambda w: 10000 + 372 * int(w.clusters)),
-        walltime=config["solving"].get("walltime", "12:00:00"),
+        runtime=config_provider("solving", "runtime", default="6h"),
    shadow:
        "minimal"
    conda:
--- a/rules/solve_myopic.smk
+++ b/rules/solve_myopic.smk
@ -1,25 +1,31 @@
-# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2023-4 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT


 rule add_existing_baseyear:
    params:
-        baseyear=config["scenario"]["planning_horizons"][0],
-        sector=config["sector"],
-        existing_capacities=config["existing_capacities"],
-        costs=config["costs"],
+        baseyear=config_provider("scenario", "planning_horizons", 0),
+        sector=config_provider("sector"),
+        existing_capacities=config_provider("existing_capacities"),
+        costs=config_provider("costs"),
    input:
        network=RESULTS
        + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-        powerplants=RESOURCES + "powerplants.csv",
-        busmap_s=RESOURCES + "busmap_elec_s{simpl}.csv",
-        busmap=RESOURCES + "busmap_elec_s{simpl}_{clusters}.csv",
-        clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv",
-        costs="data/costs_{}.csv".format(config["scenario"]["planning_horizons"][0]),
-        cop_soil_total=RESOURCES + "cop_soil_total_elec_s{simpl}_{clusters}.nc",
-        cop_air_total=RESOURCES + "cop_air_total_elec_s{simpl}_{clusters}.nc",
-        existing_heating="data/existing_infrastructure/existing_heating_raw.csv",
+        powerplants=resources("powerplants.csv"),
+        busmap_s=resources("busmap_elec_s{simpl}.csv"),
+        busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"),
+        clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"),
+        costs=lambda w: resources(
+            "costs_{}.csv".format(
+                config_provider("scenario", "planning_horizons", 0)(w)
+            )
+        ),
+        cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"),
+        cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"),
+        existing_heating_distribution=resources(
+            "existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv"
+        ),
        existing_solar="data/existing_infrastructure/solar_capacity_IRENA.csv",
        existing_onwind="data/existing_infrastructure/onwind_capacity_IRENA.csv",
        existing_offwind="data/existing_infrastructure/offwind_capacity_IRENA.csv",
@ -27,17 +33,20 @@ rule add_existing_baseyear:
        RESULTS
        + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
    wildcard_constraints:
+        # TODO: The first planning_horizon needs to be aligned across scenarios
+        # snakemake does not support passing functions to wildcard_constraints
+        # reference: https://github.com/snakemake/snakemake/issues/2703
        planning_horizons=config["scenario"]["planning_horizons"][0],  #only applies to baseyear
    threads: 1
    resources:
        mem_mb=2000,
    log:
-        LOGS
-        + "add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
+        RESULTS
+        + "logs/add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
    benchmark:
        (
-            BENCHMARKS
-            + "add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
+            RESULTS
+            + "benchmarks/add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
        )
    conda:
        "../envs/environment.yaml"
@ -45,18 +54,33 @@ rule add_existing_baseyear:
        "../scripts/add_existing_baseyear.py"


+def input_profile_tech_brownfield(w):
+    return {
+        f"profile_{tech}": resources(f"profile_{tech}.nc")
+        for tech in config_provider("electricity", "renewable_carriers")(w)
+        if tech != "hydro"
+    }
+
+
 rule add_brownfield:
    params:
-        H2_retrofit=config["sector"]["H2_retrofit"],
-        H2_retrofit_capacity_per_CH4=config["sector"]["H2_retrofit_capacity_per_CH4"],
-        threshold_capacity=config["existing_capacities"]["threshold_capacity"],
+        H2_retrofit=config_provider("sector", "H2_retrofit"),
+        H2_retrofit_capacity_per_CH4=config_provider(
+            "sector", "H2_retrofit_capacity_per_CH4"
+        ),
+        threshold_capacity=config_provider("existing_capacities", " threshold_capacity"),
+        snapshots=config_provider("snapshots"),
+        carriers=config_provider("electricity", "renewable_carriers"),
    input:
+        unpack(input_profile_tech_brownfield),
+        simplify_busmap=resources("busmap_elec_s{simpl}.csv"),
+        cluster_busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"),
        network=RESULTS
        + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
        network_p=solved_previous_horizon,  #solved network at previous time step
-        costs="data/costs_{planning_horizons}.csv",
-        cop_soil_total=RESOURCES + "cop_soil_total_elec_s{simpl}_{clusters}.nc",
-        cop_air_total=RESOURCES + "cop_air_total_elec_s{simpl}_{clusters}.nc",
+        costs=resources("costs_{planning_horizons}.csv"),
+        cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"),
+        cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"),
    output:
        RESULTS
        + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
@ -64,12 +88,12 @@ rule add_brownfield:
    resources:
        mem_mb=10000,
    log:
-        LOGS
-        + "add_brownfield_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
+        RESULTS
+        + "logs/add_brownfield_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
    benchmark:
        (
-            BENCHMARKS
-            + "add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
+            RESULTS
+            + "benchmarks/add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
        )
    conda:
        "../envs/environment.yaml"
@ -82,16 +106,17 @@ ruleorder: add_existing_baseyear > add_brownfield

 rule solve_sector_network_myopic:
    params:
-        solving=config["solving"],
-        foresight=config["foresight"],
-        planning_horizons=config["scenario"]["planning_horizons"],
-        co2_sequestration_potential=config["sector"].get(
-            "co2_sequestration_potential", 200
+        solving=config_provider("solving"),
+        foresight=config_provider("foresight"),
+        planning_horizons=config_provider("scenario", "planning_horizons"),
+        co2_sequestration_potential=config_provider(
+            "sector", "co2_sequestration_potential", default=200
        ),
+        custom_extra_functionality=input_custom_extra_functionality,
    input:
        network=RESULTS
        + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-        costs="data/costs_{planning_horizons}.csv",
+        costs=resources("costs_{planning_horizons}.csv"),
        config=RESULTS + "config.yaml",
    output:
        RESULTS
@ -99,18 +124,20 @@ rule solve_sector_network_myopic:
    shadow:
        "shallow"
    log:
-        solver=LOGS
-        + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log",
-        python=LOGS
-        + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log",
-    threads: 4
+        solver=RESULTS
+        + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log",
+        memory=RESULTS
+        + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_memory.log",
+        python=RESULTS
+        + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log",
+    threads: solver_threads
    resources:
-        mem_mb=config["solving"]["mem"],
-        walltime=config["solving"].get("walltime", "12:00:00"),
+        mem_mb=config_provider("solving", "mem_mb"),
+        runtime=config_provider("solving", "runtime", default="6h"),
    benchmark:
        (
-            BENCHMARKS
-            + "solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
+            RESULTS
+            + "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
        )
    conda:
        "../envs/environment.yaml"
--- a/rules/solve_overnight.smk
+++ b/rules/solve_overnight.smk
@ -1,16 +1,17 @@
-# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2023-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT


 rule solve_sector_network:
    params:
-        solving=config["solving"],
-        foresight=config["foresight"],
-        planning_horizons=config["scenario"]["planning_horizons"],
-        co2_sequestration_potential=config["sector"].get(
-            "co2_sequestration_potential", 200
+        solving=config_provider("solving"),
+        foresight=config_provider("foresight"),
+        planning_horizons=config_provider("scenario", "planning_horizons"),
+        co2_sequestration_potential=config_provider(
+            "sector", "co2_sequestration_potential", default=200
        ),
+        custom_extra_functionality=input_custom_extra_functionality,
    input:
        network=RESULTS
        + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
@ -21,19 +22,20 @@ rule solve_sector_network:
    shadow:
        "shallow"
    log:
-        solver=LOGS
-        + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log",
-        python=LOGS
-        + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log",
-    threads: config["solving"]["solver"].get("threads", 4)
+        solver=RESULTS
+        + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log",
+        memory=RESULTS
+        + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_memory.log",
+        python=RESULTS
+        + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log",
+    threads: solver_threads
    resources:
-        mem_mb=config["solving"]["mem"],
-        walltime=config["solving"].get("walltime", "12:00:00"),
+        mem_mb=config_provider("solving", "mem_mb"),
+        runtime=config_provider("solving", "runtime", default="6h"),
    benchmark:
        (
            RESULTS
-            + BENCHMARKS
-            + "solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
+            + "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
        )
    conda:
        "../envs/environment.yaml"
--- a/rules/solve_perfect.smk
+++ b/rules/solve_perfect.smk
@ -1,22 +1,29 @@
-# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2023-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 rule add_existing_baseyear:
    params:
-        baseyear=config["scenario"]["planning_horizons"][0],
-        sector=config["sector"],
-        existing_capacities=config["existing_capacities"],
-        costs=config["costs"],
+        baseyear=config_provider("scenario", "planning_horizons", 0),
+        sector=config_provider("sector"),
+        existing_capacities=config_provider("existing_capacities"),
+        costs=config_provider("costs"),
    input:
        network=RESULTS
        + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-        powerplants=RESOURCES + "powerplants.csv",
-        busmap_s=RESOURCES + "busmap_elec_s{simpl}.csv",
-        busmap=RESOURCES + "busmap_elec_s{simpl}_{clusters}.csv",
-        clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv",
-        costs="data/costs_{}.csv".format(config["scenario"]["planning_horizons"][0]),
-        cop_soil_total=RESOURCES + "cop_soil_total_elec_s{simpl}_{clusters}.nc",
-        cop_air_total=RESOURCES + "cop_air_total_elec_s{simpl}_{clusters}.nc",
+        powerplants=resources("powerplants.csv"),
+        busmap_s=resources("busmap_elec_s{simpl}.csv"),
+        busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"),
+        clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"),
+        costs=lambda w: resources(
+            "costs_{}.csv".format(
+                config_provider("scenario", "planning_horizons", 0)(w)
+            )
+        ),
+        cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"),
+        cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"),
+        existing_heating_distribution=resources(
+            "existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv"
+        ),
        existing_heating="data/existing_infrastructure/existing_heating_raw.csv",
        existing_solar="data/existing_infrastructure/solar_capacity_IRENA.csv",
        existing_onwind="data/existing_infrastructure/onwind_capacity_IRENA.csv",
@ -30,12 +37,12 @@ rule add_existing_baseyear:
    resources:
        mem_mb=2000,
    log:
-        LOGS
-        + "add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
+        logs(
+            "add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"
+        ),
    benchmark:
-        (
-            BENCHMARKS
-            + "add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
+        benchmarks(
+            "add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
        )
    conda:
        "../envs/environment.yaml"
@ -43,51 +50,28 @@ rule add_existing_baseyear:
        "../scripts/add_existing_baseyear.py"


-rule add_brownfield:
-    params:
-        H2_retrofit=config["sector"]["H2_retrofit"],
-        H2_retrofit_capacity_per_CH4=config["sector"]["H2_retrofit_capacity_per_CH4"],
-        threshold_capacity=config["existing_capacities"]["threshold_capacity"],
-    input:
-        network=RESULTS
-        + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-        network_p=solved_previous_horizon,  #solved network at previous time step
-        costs="data/costs_{planning_horizons}.csv",
-        cop_soil_total=RESOURCES + "cop_soil_total_elec_s{simpl}_{clusters}.nc",
-        cop_air_total=RESOURCES + "cop_air_total_elec_s{simpl}_{clusters}.nc",
-    output:
-        RESULTS
-        + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-    threads: 4
-    resources:
-        mem_mb=10000,
-    log:
-        LOGS
-        + "add_brownfield_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
-    benchmark:
-        (
-            BENCHMARKS
-            + "add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
-        )
-    conda:
-        "../envs/environment.yaml"
-    script:
-        "../scripts/add_brownfield.py"
+def input_network_year(w):
+    return {
+        f"network_{year}": RESULTS
+        + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}"
+        + f"_{year}.nc"
+        for year in config_provider("scenario", "planning_horizons")(w)[1:]
+    }


 rule prepare_perfect_foresight:
+    params:
+        costs=config_provider("costs"),
+        time_resolution=config_provider("clustering", "temporal", "sector"),
    input:
-        **{
-            f"network_{year}": RESULTS
-            + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_"
-            + f"{year}.nc"
-            for year in config["scenario"]["planning_horizons"][1:]
-        },
+        unpack(input_network_year),
        brownfield_network=lambda w: (
            RESULTS
            + "prenetworks-brownfield/"
            + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_"
-            + "{}.nc".format(str(config["scenario"]["planning_horizons"][0]))
+            + "{}.nc".format(
+                str(config_provider("scenario", "planning_horizons", 0)(w))
+            )
        ),
    output:
        RESULTS
@ -96,12 +80,12 @@ rule prepare_perfect_foresight:
    resources:
        mem_mb=10000,
    log:
-        LOGS
-        + "prepare_perfect_foresight{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}.log",
+        logs(
+            "prepare_perfect_foresight{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}.log"
+        ),
    benchmark:
-        (
-            BENCHMARKS
-            + "prepare_perfect_foresight{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}"
+        benchmarks(
+            "prepare_perfect_foresight{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}"
        )
    conda:
        "../envs/environment.yaml"
@ -111,24 +95,25 @@ rule prepare_perfect_foresight:

 rule solve_sector_network_perfect:
    params:
-        solving=config["solving"],
-        foresight=config["foresight"],
-        sector=config["sector"],
-        planning_horizons=config["scenario"]["planning_horizons"],
-        co2_sequestration_potential=config["sector"].get(
-            "co2_sequestration_potential", 200
+        solving=config_provider("solving"),
+        foresight=config_provider("foresight"),
+        sector=config_provider("sector"),
+        planning_horizons=config_provider("scenario", "planning_horizons"),
+        co2_sequestration_potential=config_provider(
+            "sector", "co2_sequestration_potential", default=200
        ),
+        custom_extra_functionality=input_custom_extra_functionality,
    input:
        network=RESULTS
        + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
-        costs="data/costs_2030.csv",
+        costs=resources("costs_2030.csv"),
        config=RESULTS + "config.yaml",
    output:
        RESULTS
        + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
-    threads: 4
+    threads: solver_threads
    resources:
-        mem_mb=config["solving"]["mem"],
+        mem_mb=config_provider("solving", "mem"),
    shadow:
        "shallow"
    log:
@ -140,8 +125,8 @@ rule solve_sector_network_perfect:
        + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_memory.log",
    benchmark:
        (
-            BENCHMARKS
-            + "solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years}"
+            RESULTS
+            + "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years}"
        )
    conda:
        "../envs/environment.yaml"
@ -149,18 +134,22 @@ rule solve_sector_network_perfect:
        "../scripts/solve_network.py"


+def input_networks_make_summary_perfect(w):
+    return {
+        f"networks_{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}": RESULTS
+        + f"postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc"
+        for simpl in config_provider("scenario", "simpl")(w)
+        for clusters in config_provider("scenario", "clusters")(w)
+        for opts in config_provider("scenario", "opts")(w)
+        for sector_opts in config_provider("scenario", "sector_opts")(w)
+        for ll in config_provider("scenario", "ll")(w)
+    }
+
+
 rule make_summary_perfect:
    input:
-        **{
-            f"networks_{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}": RESULTS
-            + f"postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc"
-            for simpl in config["scenario"]["simpl"]
-            for clusters in config["scenario"]["clusters"]
-            for opts in config["scenario"]["opts"]
-            for sector_opts in config["scenario"]["sector_opts"]
-            for ll in config["scenario"]["ll"]
-        },
-        costs="data/costs_2020.csv",
+        unpack(input_networks_make_summary_perfect),
+        costs=resources("costs_2020.csv"),
    output:
        nodal_costs=RESULTS + "csvs/nodal_costs.csv",
        nodal_capacities=RESULTS + "csvs/nodal_capacities.csv",
@ -182,13 +171,10 @@ rule make_summary_perfect:
    resources:
        mem_mb=10000,
    log:
-        LOGS + "make_summary_perfect.log",
+        logs("make_summary_perfect.log"),
    benchmark:
-        (BENCHMARKS + "make_summary_perfect")
+        benchmarks("make_summary_perfect")
    conda:
        "../envs/environment.yaml"
    script:
        "../scripts/make_summary_perfect.py"
-
-
-ruleorder: add_existing_baseyear > add_brownfield
--- a/rules/validate.smk
+++ b/rules/validate.smk
@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2023-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT

@ -17,12 +17,12 @@ rule build_electricity_production:
    The data is used for validation of the optimization results.
    """
    params:
-        snapshots=config["snapshots"],
-        countries=config["countries"],
+        snapshots=config_provider("snapshots"),
+        countries=config_provider("countries"),
    output:
-        RESOURCES + "historical_electricity_production.csv",
+        resources("historical_electricity_production.csv"),
    log:
-        LOGS + "build_electricity_production.log",
+        logs("build_electricity_production.log"),
    resources:
        mem_mb=5000,
    script:
@ -35,14 +35,14 @@ rule build_cross_border_flows:
    The data is used for validation of the optimization results.
    """
    params:
-        snapshots=config["snapshots"],
-        countries=config["countries"],
+        snapshots=config_provider("snapshots"),
+        countries=config_provider("countries"),
    input:
-        network=RESOURCES + "networks/base.nc",
+        network=resources("networks/base.nc"),
    output:
-        RESOURCES + "historical_cross_border_flows.csv",
+        resources("historical_cross_border_flows.csv"),
    log:
-        LOGS + "build_cross_border_flows.log",
+        logs("build_cross_border_flows.log"),
    resources:
        mem_mb=5000,
    script:
@ -55,12 +55,12 @@ rule build_electricity_prices:
    The data is used for validation of the optimization results.
    """
    params:
-        snapshots=config["snapshots"],
-        countries=config["countries"],
+        snapshots=config_provider("snapshots"),
+        countries=config_provider("countries"),
    output:
-        RESOURCES + "historical_electricity_prices.csv",
+        resources("historical_electricity_prices.csv"),
    log:
-        LOGS + "build_electricity_prices.log",
+        logs("build_electricity_prices.log"),
    resources:
        mem_mb=5000,
    script:
@ -70,7 +70,7 @@ rule build_electricity_prices:
 rule plot_validation_electricity_production:
    input:
        network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
-        electricity_production=RESOURCES + "historical_electricity_production.csv",
+        electricity_production=resources("historical_electricity_production.csv"),
    output:
        **{
            plot: RESULTS
@ -85,10 +85,10 @@ rule plot_validation_electricity_production:

 rule plot_validation_cross_border_flows:
    params:
-        countries=config["countries"],
+        countries=config_provider("countries"),
    input:
        network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
-        cross_border_flows=RESOURCES + "historical_cross_border_flows.csv",
+        cross_border_flows=resources("historical_cross_border_flows.csv"),
    output:
        **{
            plot: RESULTS
@ -104,7 +104,7 @@ rule plot_validation_cross_border_flows:
 rule plot_validation_electricity_prices:
    input:
        network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
-        electricity_prices=RESOURCES + "historical_electricity_prices.csv",
+        electricity_prices=resources("historical_electricity_prices.csv"),
    output:
        **{
            plot: RESULTS
--- a/scripts/init.py
+++ b/scripts/init.py
@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
--- a/Show More
+++ b/Show More