merge master

.github/workflows/ci.yaml

@@ -19,7 +19,6 @@ on:
   - cron: "0 5 * * TUE"
 
 env:
-  CONDA_CACHE_NUMBER: 1 # Change this value to manually reset the environment cache
   DATA_CACHE_NUMBER: 2
 
 jobs:
@@ -27,22 +26,12 @@ jobs:
 
     strategy:
       fail-fast: false
+      max-parallel: 3
       matrix:
-        include:
-          # Matrix required to handle caching with Mambaforge
-        - os: ubuntu-latest
-          label: ubuntu-latest
-          prefix: /usr/share/miniconda3/envs/pypsa-eur
-
-        - os: macos-latest
-          label: macos-latest
-          prefix: /Users/runner/miniconda3/envs/pypsa-eur
-
-        - os: windows-latest
-          label: windows-latest
-          prefix: C:\Miniconda3\envs\pypsa-eur
-
-    name: ${{ matrix.label }}
+        os:
+        - ubuntu-latest
+        - macos-latest
+        - windows-latest
 
     runs-on: ${{ matrix.os }}
 
@@ -60,24 +49,25 @@ jobs:
     - name: Add solver to environment
       run: |
         echo -e "- glpk\n- ipopt<3.13.3" >> envs/environment.yaml
-      if: ${{ matrix.label }} == 'windows-latest'
+      if: ${{ matrix.os }} == 'windows-latest'
 
     - name: Add solver to environment
       run: |
         echo -e "- glpk\n- ipopt" >> envs/environment.yaml
-      if: ${{ matrix.label }} != 'windows-latest'
+      if: ${{ matrix.os }} != 'windows-latest'
 
-    - name: Setup Mambaforge
-      uses: conda-incubator/setup-miniconda@v2
+    - name: Setup micromamba
+      uses: mamba-org/setup-micromamba@v1
       with:
-        miniforge-variant: Mambaforge
-        miniforge-version: latest
-        activate-environment: pypsa-eur
-        use-mamba: true
+        micromamba-version: latest
+        environment-file: envs/environment.yaml
+        log-level: debug
+        init-shell: bash
+        cache-environment: true
+        cache-downloads: true
 
     - name: Set cache dates
       run: |
-        echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV
         echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV
 
     - name: Cache data and cutouts folders
@@ -88,21 +78,8 @@ jobs:
           cutouts
         key: data-cutouts-${{ env.WEEK }}-${{ env.DATA_CACHE_NUMBER }}
 
-    - name: Create environment cache
-      uses: actions/cache@v3
-      id: cache
-      with:
-        path: ${{ matrix.prefix }}
-        key: ${{ matrix.label }}-conda-${{ env.DATE }}-${{ env.CONDA_CACHE_NUMBER }}
-
-    - name: Update environment due to outdated or unavailable cache
-      run: mamba env update -n pypsa-eur -f envs/environment.yaml
-      if: steps.cache.outputs.cache-hit != 'true'
-
     - name: Test snakemake workflow
       run: |
-        conda activate pypsa-eur
-        conda list
         snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime
         snakemake -call all --configfile config/test/config.overnight.yaml --rerun-triggers=mtime
         snakemake -call all --configfile config/test/config.myopic.yaml --rerun-triggers=mtime

.pre-commit-config.yaml

@@ -87,6 +87,6 @@ repos:
 
   # Check for FSFE REUSE compliance (licensing)
 - repo: https://github.com/fsfe/reuse-tool
-  rev: v2.0.0
+  rev: v2.1.0
   hooks:
   - id: reuse

.sync-send

@@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: CC0-1.0
+
+rules
+scripts
+config
+config/test
+envs
+matplotlibrc
+Snakefile

.syncignore-receive

@@ -1,21 +0,0 @@
-# SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors
-#
-# SPDX-License-Identifier: CC0-1.0
-
-.snakemake
-.git
-.pytest_cache
-.ipynb_checkpoints
-.vscode
-.DS_Store
-__pycache__
-*.pyc
-*.pyo
-*.ipynb
-notebooks
-doc
-cutouts
-data
-benchmarks
-*.nc
-configs

.syncignore-send

@@ -1,23 +0,0 @@
-# SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors
-#
-# SPDX-License-Identifier: CC0-1.0
-
-.snakemake
-.git
-.pytest_cache
-.ipynb_checkpoints
-.vscode
-.DS_Store
-__pycache__
-*.pyc
-*.pyo
-*.ipynb
-notebooks
-benchmarks
-logs
-resources*
-results
-networks*
-cutouts
-data/bundle
-doc

CITATION.cff

@@ -6,7 +6,7 @@ cff-version: 1.1.0
 message: "If you use this package, please cite it in the following way."
 title: "PyPSA-Eur: An open sector-coupled optimisation model of the European energy system"
 repository: https://github.com/pypsa/pypsa-eur
-version: 0.8.0
+version: 0.8.1
 license: MIT
 authors:
   - family-names: Brown

README.md

@@ -9,7 +9,7 @@ SPDX-License-Identifier: CC-BY-4.0
 ![Size](https://img.shields.io/github/repo-size/pypsa/pypsa-eur)
 [![Zenodo PyPSA-Eur](https://zenodo.org/badge/DOI/10.5281/zenodo.3520874.svg)](https://doi.org/10.5281/zenodo.3520874)
 [![Zenodo PyPSA-Eur-Sec](https://zenodo.org/badge/DOI/10.5281/zenodo.3938042.svg)](https://doi.org/10.5281/zenodo.3938042)
-[![Snakemake](https://img.shields.io/badge/snakemake-≥5.0.0-brightgreen.svg?style=flat)](https://snakemake.readthedocs.io)
+[![Snakemake](https://img.shields.io/badge/snakemake-≥7.7.0-brightgreen.svg?style=flat)](https://snakemake.readthedocs.io)
 [![REUSE status](https://api.reuse.software/badge/github.com/pypsa/pypsa-eur)](https://api.reuse.software/info/github.com/pypsa/pypsa-eur)
 [![Stack Exchange questions](https://img.shields.io/stackexchange/stackoverflow/t/pypsa)](https://stackoverflow.com/questions/tagged/pypsa)
 
@@ -35,17 +35,18 @@ The model is designed to be imported into the open toolbox
 [PyPSA](https://github.com/PyPSA/PyPSA).
 
 **WARNING**: PyPSA-Eur is under active development and has several
-[limitations](https://pypsa-eur.readthedocs.io/en/latest/limitations.html)
-which you should understand before using the model. The github repository
+[limitations](https://pypsa-eur.readthedocs.io/en/latest/limitations.html) which
+you should understand before using the model. The github repository
 [issues](https://github.com/PyPSA/pypsa-eur/issues) collect known topics we are
 working on (please feel free to help or make suggestions). The
 [documentation](https://pypsa-eur.readthedocs.io/) remains somewhat patchy. You
-can find showcases of the model's capabilities in the preprint [Benefits of a
-Hydrogen Network in Europe](https://arxiv.org/abs/2207.05816), a [paper in Joule
-with a description of the industry sector](https://arxiv.org/abs/2109.09563), or
-in [a 2021 presentation at EMP-E](https://nworbmot.org/energy/brown-empe.pdf).
-We cannot support this model if you choose to use it. We do not recommend to use
-the full resolution network model for simulations. At high granularity the
+can find showcases of the model's capabilities in the Joule paper [The potential
+role of a hydrogen network in
+Europe](https://doi.org/10.1016/j.joule.2023.06.016), another [paper in Joule
+with a description of the industry
+sector](https://doi.org/10.1016/j.joule.2022.04.016), or in [a 2021 presentation
+at EMP-E](https://nworbmot.org/energy/brown-empe.pdf). We do not recommend to
+use the full resolution network model for simulations. At high granularity the
 assignment of loads and generators to the nearest network node may not be a
 correct assumption, depending on the topology of the underlying distribution
 grid, and local grid bottlenecks may cause unrealistic load-shedding or

Snakefile

@@ -40,7 +40,7 @@ localrules:
 
 wildcard_constraints:
     simpl="[a-zA-Z0-9]*",
-    clusters="[0-9]+m?|all",
+    clusters="[0-9]+(m|c)?|all",
     ll="(v|c)([0-9\.]+|opt)",
     opts="[-+a-zA-Z0-9\.]*",
     sector_opts="[-+a-zA-Z0-9\.\s]*",
@@ -53,6 +53,7 @@ include: "rules/build_electricity.smk"
 include: "rules/build_sector.smk"
 include: "rules/solve_electricity.smk"
 include: "rules/postprocess.smk"
+include: "rules/validate.smk"
 
 
 if config["foresight"] == "overnight":
@@ -103,3 +104,14 @@ rule doc:
         directory("doc/_build"),
     shell:
         "make -C doc html"
+
+
+rule sync:
+    params:
+        cluster=f"{config['remote']['ssh']}:{config['remote']['path']}",
+    shell:
+        """
+        rsync -uvarh --ignore-missing-args --files-from=.sync-send . {params.cluster}
+        rsync -uvarh --no-g {params.cluster}/results . || echo "No results directory, skipping rsync"
+        rsync -uvarh --no-g {params.cluster}/logs . || echo "No logs directory, skipping rsync"
+        """

config/config.default.yaml

@@ -3,13 +3,21 @@
 # SPDX-License-Identifier: CC0-1.0
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#top-level-configuration
-version: 0.8.0
+version: 0.8.1
 tutorial: false
 
 logging:
   level: INFO
   format: '%(levelname)s:%(name)s:%(message)s'
 
+private:
+  keys:
+    entsoe_api:
+
+remote:
+  ssh: ""
+  path: ""
+
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#run
 run:
   name: ""
@@ -18,7 +26,7 @@ run:
   shared_cutouts: true
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight
-foresight: overnight
+foresight: perfect
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#scenario
 # Wildcard docs in https://pypsa-eur.readthedocs.io/en/latest/wildcards.html
@@ -26,21 +34,22 @@ scenario:
   simpl:
   - ''
   ll:
+  - v1.0
   - v1.5
   clusters:
   - 37
   - 128
-  - 256
-  - 512
-  - 1024
+#  - 256
+#  - 512
+  #- 1024
   opts:
   - ''
   sector_opts:
   - Co2L0-3H-T-H-B-I-A-solar+p3-dist1
   planning_horizons:
-  # - 2020
-  # - 2030
-  # - 2040
+  - 2020
+  - 2030
+  - 2040
   - 2050
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#countries
@@ -209,10 +218,14 @@ renewable:
     carriers: [ror, PHS, hydro]
     PHS_max_hours: 6
     hydro_max_hours: "energy_capacity_totals_by_country" # one of energy_capacity_totals_by_country, estimate_by_large_installations or a float
+    flatten_dispatch: false
+    flatten_dispatch_buffer: 0.2
     clip_min_inflow: 1.0
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#conventional
 conventional:
+  unit_commitment: false
+  dynamic_fuel_price: false
   nuclear:
     p_max_pu: "data/nuclear_p_max_pu.csv" # float of file name
 
@@ -227,6 +240,12 @@ lines:
   max_extension: .inf
   length_factor: 1.25
   under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity
+  dynamic_line_rating:
+    activate: false
+    cutout: europe-2013-era5
+    correction_factor: 0.95
+    max_voltage_difference: false
+    max_line_rating: false
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#links
 links:
@@ -528,7 +547,7 @@ industry:
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#costs
 costs:
   year: 2030
-  version: v0.5.0
+  version: v0.6.0
   rooftop_share: 0.14  # based on the potentials, assuming  (0.1 kW/m2 and 10 m2/person)
   fill_values:
     FOM: 0
@@ -568,16 +587,12 @@ clustering:
     algorithm: kmeans
     feature: solar+onwind-time
     exclude_carriers: []
+    consider_efficiency_classes: false
   aggregation_strategies:
     generators:
-      p_nom_max: sum
-      p_nom_min: sum
-      p_min_pu: mean
-      marginal_cost: mean
       committable: any
       ramp_limit_up: max
       ramp_limit_down: max
-      efficiency: mean
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#solving
 solving:
@@ -585,13 +600,17 @@ solving:
   options:
     clip_p_max_pu: 1.e-2
     load_shedding: false
-    transmission_losses: 0
     noisy_costs: true
     skip_iterations: true
+    rolling_horizon: false
+    seed: 123
+    # options that go into the optimize function
     track_iterations: false
     min_iterations: 4
     max_iterations: 6
-    seed: 123
+    transmission_losses: 0
+    linearized_unit_commitment: true
+    horizon: 365
 
   solver:
     name: gurobi
@@ -619,7 +638,6 @@ solving:
       AggFill: 0
       PreDual: 0
       GURO_PAR_BARDENSETHRESH: 200
-      seed: 10              # Consistent seed for all plattforms
     gurobi-numeric-focus:
       name: gurobi
       NumericFocus: 3       # Favour numeric stability over speed
@@ -652,6 +670,7 @@ solving:
     glpk-default: {} # Used in CI
 
   mem: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2
+  walltime: "12:00:00"
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#plotting
 plotting:
@@ -682,6 +701,8 @@ plotting:
     H2: "Hydrogen Storage"
     lines: "Transmission Lines"
     ror: "Run of River"
+    ac: "AC"
+    dc: "DC"
 
   tech_colors:
     # wind

config/config.validation.yaml

@@ -0,0 +1,98 @@
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: CC0-1.0
+run:
+  name: "validation"
+
+scenario:
+  ll:
+  - v1.0
+  clusters:
+  - 37
+  opts:
+  - 'Ept'
+
+snapshots:
+  start: "2019-01-01"
+  end: "2020-01-01"
+  inclusive: 'left'
+
+enable:
+  retrieve_cutout: false
+
+electricity:
+  co2limit: 1e9
+
+  extendable_carriers:
+    Generator: []
+    StorageUnit: []
+    Store: []
+    Link: []
+
+  powerplants_filter: not (DateOut < 2019)
+
+  conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass]
+  renewable_carriers: [solar, onwind, offwind-ac, offwind-dc, hydro]
+
+  estimate_renewable_capacities:
+    year: 2019
+
+atlite:
+  default_cutout: europe-2019-era5
+  cutouts:
+    europe-2019-era5:
+      module: era5
+      x: [-12., 35.]
+      y: [33., 72]
+      dx: 0.3
+      dy: 0.3
+      time: ['2019', '2019']
+
+renewable:
+  onwind:
+    cutout: europe-2019-era5
+  offwind-ac:
+    cutout: europe-2019-era5
+  offwind-dc:
+    cutout: europe-2019-era5
+  solar:
+    cutout: europe-2019-era5
+  hydro:
+    cutout: europe-2019-era5
+    flatten_dispatch: 0.01
+
+conventional:
+  unit_commitment: false
+  dynamic_fuel_price: true
+  nuclear:
+    p_max_pu: "data/nuclear_p_max_pu.csv"
+  biomass:
+    p_max_pu: 0.65
+
+load:
+  power_statistics: false
+
+lines:
+  s_max_pu: 0.23
+  under_construction: 'remove'
+
+links:
+  include_tyndp: false
+
+costs:
+  year: 2020
+  emission_prices:
+    co2: 25
+
+clustering:
+  simplify_network:
+    exclude_carriers: [oil, coal, lignite, OCGT, CCGT]
+  cluster_network:
+    consider_efficiency_classes: true
+
+solving:
+  options:
+    load_shedding: true
+    rolling_horizon: false
+    horizon: 1000
+    overlap: 48

config/test/config.electricity.yaml

@@ -60,6 +60,12 @@ renewable:
 clustering:
   exclude_carriers: ["OCGT", "offwind-ac", "coal"]
 
+lines:
+  dynamic_line_rating:
+    activate: true
+    cutout: be-03-2013-era5
+    max_line_rating: 1.3
+
 
 solving:
   solver:

data/costs.csv

@@ -1,195 +0,0 @@
-technology,year,parameter,value,unit,source
-solar-rooftop,2030,discount rate,0.04,per unit,standard for decentral
-onwind,2030,lifetime,30,years,DEA https://ens.dk/en/our-services/projections-and-models/technology-data
-offwind,2030,lifetime,30,years,DEA https://ens.dk/en/our-services/projections-and-models/technology-data
-solar,2030,lifetime,25,years,IEA2010
-solar-rooftop,2030,lifetime,25,years,IEA2010
-solar-utility,2030,lifetime,25,years,IEA2010
-PHS,2030,lifetime,80,years,IEA2010
-hydro,2030,lifetime,80,years,IEA2010
-ror,2030,lifetime,80,years,IEA2010
-OCGT,2030,lifetime,30,years,IEA2010
-nuclear,2030,lifetime,45,years,ECF2010 in DIW DataDoc http://hdl.handle.net/10419/80348
-CCGT,2030,lifetime,30,years,IEA2010
-coal,2030,lifetime,40,years,IEA2010
-lignite,2030,lifetime,40,years,IEA2010
-geothermal,2030,lifetime,40,years,IEA2010
-biomass,2030,lifetime,30,years,ECF2010 in DIW DataDoc http://hdl.handle.net/10419/80348
-oil,2030,lifetime,30,years,ECF2010 in DIW DataDoc http://hdl.handle.net/10419/80348
-onwind,2030,investment,1040,EUR/kWel,DEA https://ens.dk/en/our-services/projections-and-models/technology-data
-offwind,2030,investment,1640,EUR/kWel,DEA https://ens.dk/en/our-services/projections-and-models/technology-data
-offwind-ac-station,2030,investment,250,EUR/kWel,DEA https://ens.dk/en/our-services/projections-and-models/technology-data
-offwind-ac-connection-submarine,2030,investment,2685,EUR/MW/km,DEA https://ens.dk/en/our-services/projections-and-models/technology-data
-offwind-ac-connection-underground,2030,investment,1342,EUR/MW/km,DEA https://ens.dk/en/our-services/projections-and-models/technology-data
-offwind-dc-station,2030,investment,400,EUR/kWel,Haertel 2017; assuming one onshore and one offshore node + 13% learning reduction
-offwind-dc-connection-submarine,2030,investment,2000,EUR/MW/km,DTU report based on Fig 34 of https://ec.europa.eu/energy/sites/ener/files/documents/2014_nsog_report.pdf
-offwind-dc-connection-underground,2030,investment,1000,EUR/MW/km,Haertel 2017; average + 13% learning reduction
-solar,2030,investment,600,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348
-biomass,2030,investment,2209,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348
-geothermal,2030,investment,3392,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348
-coal,2030,investment,1300,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348 PC (Advanced/SuperC)
-lignite,2030,investment,1500,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348
-solar-rooftop,2030,investment,725,EUR/kWel,ETIP PV
-solar-utility,2030,investment,425,EUR/kWel,ETIP PV
-PHS,2030,investment,2000,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348
-hydro,2030,investment,2000,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348
-ror,2030,investment,3000,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348
-OCGT,2030,investment,400,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348
-nuclear,2030,investment,6000,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348
-CCGT,2030,investment,800,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348
-oil,2030,investment,400,EUR/kWel,DIW DataDoc http://hdl.handle.net/10419/80348
-onwind,2030,FOM,2.450549,%/year,DEA https://ens.dk/en/our-services/projections-and-models/technology-data
-offwind,2030,FOM,2.304878,%/year,DEA https://ens.dk/en/our-services/projections-and-models/technology-data
-solar,2030,FOM,4.166667,%/year,DIW DataDoc http://hdl.handle.net/10419/80348
-solar-rooftop,2030,FOM,2,%/year,ETIP PV
-solar-utility,2030,FOM,3,%/year,ETIP PV
-biomass,2030,FOM,4.526935,%/year,DIW DataDoc http://hdl.handle.net/10419/80348
-geothermal,2030,FOM,2.358491,%/year,DIW DataDoc http://hdl.handle.net/10419/80348
-coal,2030,FOM,1.923076,%/year,DIW DataDoc http://hdl.handle.net/10419/80348 PC (Advanced/SuperC)
-lignite,2030,FOM,2.0,%/year,DIW DataDoc http://hdl.handle.net/10419/80348 PC (Advanced/SuperC)
-oil,2030,FOM,1.5,%/year,DIW DataDoc http://hdl.handle.net/10419/80348
-PHS,2030,FOM,1,%/year,DIW DataDoc http://hdl.handle.net/10419/80348
-hydro,2030,FOM,1,%/year,DIW DataDoc http://hdl.handle.net/10419/80348
-ror,2030,FOM,2,%/year,DIW DataDoc http://hdl.handle.net/10419/80348
-CCGT,2030,FOM,2.5,%/year,DIW DataDoc http://hdl.handle.net/10419/80348
-OCGT,2030,FOM,3.75,%/year,DIW DataDoc http://hdl.handle.net/10419/80348
-onwind,2030,VOM,2.3,EUR/MWhel,DEA https://ens.dk/en/our-services/projections-and-models/technology-data
-offwind,2030,VOM,2.7,EUR/MWhel,DEA https://ens.dk/en/our-services/projections-and-models/technology-data
-solar,2030,VOM,0.01,EUR/MWhel,RES costs made up to fix curtailment order
-coal,2030,VOM,6,EUR/MWhel,DIW DataDoc http://hdl.handle.net/10419/80348 PC (Advanced/SuperC)
-lignite,2030,VOM,7,EUR/MWhel,DIW DataDoc http://hdl.handle.net/10419/80348
-CCGT,2030,VOM,4,EUR/MWhel,DIW DataDoc http://hdl.handle.net/10419/80348
-OCGT,2030,VOM,3,EUR/MWhel,DIW DataDoc http://hdl.handle.net/10419/80348
-nuclear,2030,VOM,8,EUR/MWhel,DIW DataDoc http://hdl.handle.net/10419/80348
-gas,2030,fuel,21.6,EUR/MWhth,IEA2011b
-uranium,2030,fuel,3,EUR/MWhth,DIW DataDoc http://hdl.handle.net/10419/80348
-oil,2030,VOM,3,EUR/MWhel,DIW DataDoc http://hdl.handle.net/10419/80348
-nuclear,2030,fuel,3,EUR/MWhth,IEA2011b
-biomass,2030,fuel,7,EUR/MWhth,IEA2011b
-coal,2030,fuel,8.4,EUR/MWhth,IEA2011b
-lignite,2030,fuel,2.9,EUR/MWhth,IEA2011b
-oil,2030,fuel,50,EUR/MWhth,IEA WEM2017 97USD/boe = http://www.iea.org/media/weowebsite/2017/WEM_Documentation_WEO2017.pdf
-PHS,2030,efficiency,0.75,per unit,DIW DataDoc http://hdl.handle.net/10419/80348
-hydro,2030,efficiency,0.9,per unit,DIW DataDoc http://hdl.handle.net/10419/80348
-ror,2030,efficiency,0.9,per unit,DIW DataDoc http://hdl.handle.net/10419/80348
-OCGT,2030,efficiency,0.39,per unit,DIW DataDoc http://hdl.handle.net/10419/80348
-CCGT,2030,efficiency,0.5,per unit,DIW DataDoc http://hdl.handle.net/10419/80348
-biomass,2030,efficiency,0.468,per unit,DIW DataDoc http://hdl.handle.net/10419/80348
-geothermal,2030,efficiency,0.239,per unit,DIW DataDoc http://hdl.handle.net/10419/80348
-nuclear,2030,efficiency,0.337,per unit,DIW DataDoc http://hdl.handle.net/10419/80348
-gas,2030,CO2 intensity,0.187,tCO2/MWth,https://www.eia.gov/environment/emissions/co2_vol_mass.php
-coal,2030,efficiency,0.464,per unit,DIW DataDoc http://hdl.handle.net/10419/80348 PC (Advanced/SuperC)
-lignite,2030,efficiency,0.447,per unit,DIW DataDoc http://hdl.handle.net/10419/80348
-oil,2030,efficiency,0.393,per unit,DIW DataDoc http://hdl.handle.net/10419/80348 CT
-coal,2030,CO2 intensity,0.354,tCO2/MWth,https://www.eia.gov/environment/emissions/co2_vol_mass.php
-lignite,2030,CO2 intensity,0.334,tCO2/MWth,https://www.eia.gov/environment/emissions/co2_vol_mass.php
-oil,2030,CO2 intensity,0.248,tCO2/MWth,https://www.eia.gov/environment/emissions/co2_vol_mass.php
-geothermal,2030,CO2 intensity,0.026,tCO2/MWth,https://www.eia.gov/environment/emissions/co2_vol_mass.php
-electrolysis,2030,investment,350,EUR/kWel,Palzer Thesis
-electrolysis,2030,FOM,4,%/year,NREL http://www.nrel.gov/docs/fy09osti/45873.pdf; budischak2013
-electrolysis,2030,lifetime,18,years,NREL http://www.nrel.gov/docs/fy09osti/45873.pdf; budischak2013
-electrolysis,2030,efficiency,0.8,per unit,NREL http://www.nrel.gov/docs/fy09osti/45873.pdf; budischak2013
-fuel cell,2030,investment,339,EUR/kWel,NREL http://www.nrel.gov/docs/fy09osti/45873.pdf; budischak2013
-fuel cell,2030,FOM,3,%/year,NREL http://www.nrel.gov/docs/fy09osti/45873.pdf; budischak2013
-fuel cell,2030,lifetime,20,years,NREL http://www.nrel.gov/docs/fy09osti/45873.pdf; budischak2013
-fuel cell,2030,efficiency,0.58,per unit,NREL http://www.nrel.gov/docs/fy09osti/45873.pdf; budischak2013 conservative 2020
-hydrogen storage,2030,investment,11.2,USD/kWh,budischak2013
-hydrogen storage,2030,lifetime,20,years,budischak2013
-hydrogen underground storage,2030,investment,0.5,EUR/kWh,maximum from https://www.nrel.gov/docs/fy10osti/46719.pdf
-hydrogen underground storage,2030,lifetime,40,years,http://www.acatech.de/fileadmin/user_upload/Baumstruktur_nach_Website/Acatech/root/de/Publikationen/Materialien/ESYS_Technologiesteckbrief_Energiespeicher.pdf
-H2 pipeline,2030,investment,267,EUR/MW/km,Welder et al https://doi.org/10.1016/j.ijhydene.2018.12.156
-H2 pipeline,2030,lifetime,40,years,Krieg2012 http://juser.fz-juelich.de/record/136392/files/Energie%26Umwelt_144.pdf
-H2 pipeline,2030,FOM,5,%/year,Krieg2012 http://juser.fz-juelich.de/record/136392/files/Energie%26Umwelt_144.pdf
-H2 pipeline,2030,efficiency,0.98,per unit,Krieg2012 http://juser.fz-juelich.de/record/136392/files/Energie%26Umwelt_144.pdf
-methanation,2030,investment,1000,EUR/kWH2,Schaber thesis
-methanation,2030,lifetime,25,years,Schaber thesis
-methanation,2030,FOM,3,%/year,Schaber thesis
-methanation,2030,efficiency,0.6,per unit,Palzer; Breyer for DAC
-helmeth,2030,investment,1000,EUR/kW,no source
-helmeth,2030,lifetime,25,years,no source
-helmeth,2030,FOM,3,%/year,no source
-helmeth,2030,efficiency,0.8,per unit,HELMETH press release
-DAC,2030,investment,250,EUR/(tCO2/a),Fasihi/Climeworks
-DAC,2030,lifetime,30,years,Fasihi
-DAC,2030,FOM,4,%/year,Fasihi
-battery inverter,2030,investment,411,USD/kWel,budischak2013
-battery inverter,2030,lifetime,20,years,budischak2013
-battery inverter,2030,efficiency,0.9,per unit charge/discharge,budischak2013; Lund and Kempton (2008) http://dx.doi.org/10.1016/j.enpol.2008.06.007
-battery inverter,2030,FOM,3,%/year,budischak2013
-battery storage,2030,investment,192,USD/kWh,budischak2013
-battery storage,2030,lifetime,15,years,budischak2013
-decentral air-sourced heat pump,2030,investment,1050,EUR/kWth,HP; Palzer thesis
-decentral air-sourced heat pump,2030,lifetime,20,years,HP; Palzer thesis
-decentral air-sourced heat pump,2030,FOM,3.5,%/year,Palzer thesis
-decentral air-sourced heat pump,2030,efficiency,3,per unit,default for costs
-decentral air-sourced heat pump,2030,discount rate,0.04,per unit,Palzer thesis
-decentral ground-sourced heat pump,2030,investment,1400,EUR/kWth,Palzer thesis
-decentral ground-sourced heat pump,2030,lifetime,20,years,Palzer thesis
-decentral ground-sourced heat pump,2030,FOM,3.5,%/year,Palzer thesis
-decentral ground-sourced heat pump,2030,efficiency,4,per unit,default for costs
-decentral ground-sourced heat pump,2030,discount rate,0.04,per unit,Palzer thesis
-central air-sourced heat pump,2030,investment,700,EUR/kWth,Palzer thesis
-central air-sourced heat pump,2030,lifetime,20,years,Palzer thesis
-central air-sourced heat pump,2030,FOM,3.5,%/year,Palzer thesis
-central air-sourced heat pump,2030,efficiency,3,per unit,default for costs
-retrofitting I,2030,discount rate,0.04,per unit,Palzer thesis
-retrofitting I,2030,lifetime,50,years,Palzer thesis
-retrofitting I,2030,FOM,1,%/year,Palzer thesis
-retrofitting I,2030,investment,50,EUR/m2/fraction reduction,Palzer thesis
-retrofitting II,2030,discount rate,0.04,per unit,Palzer thesis
-retrofitting II,2030,lifetime,50,years,Palzer thesis
-retrofitting II,2030,FOM,1,%/year,Palzer thesis
-retrofitting II,2030,investment,250,EUR/m2/fraction reduction,Palzer thesis
-water tank charger,2030,efficiency,0.9,per unit,HP
-water tank discharger,2030,efficiency,0.9,per unit,HP
-decentral water tank storage,2030,investment,860,EUR/m3,IWES Interaktion
-decentral water tank storage,2030,FOM,1,%/year,HP
-decentral water tank storage,2030,lifetime,20,years,HP
-decentral water tank storage,2030,discount rate,0.04,per unit,Palzer thesis
-central water tank storage,2030,investment,30,EUR/m3,IWES Interaktion
-central water tank storage,2030,FOM,1,%/year,HP
-central water tank storage,2030,lifetime,40,years,HP
-decentral resistive heater,2030,investment,100,EUR/kWhth,Schaber thesis
-decentral resistive heater,2030,lifetime,20,years,Schaber thesis
-decentral resistive heater,2030,FOM,2,%/year,Schaber thesis
-decentral resistive heater,2030,efficiency,0.9,per unit,Schaber thesis
-decentral resistive heater,2030,discount rate,0.04,per unit,Palzer thesis
-central resistive heater,2030,investment,100,EUR/kWhth,Schaber thesis
-central resistive heater,2030,lifetime,20,years,Schaber thesis
-central resistive heater,2030,FOM,2,%/year,Schaber thesis
-central resistive heater,2030,efficiency,0.9,per unit,Schaber thesis
-decentral gas boiler,2030,investment,175,EUR/kWhth,Palzer thesis
-decentral gas boiler,2030,lifetime,20,years,Palzer thesis
-decentral gas boiler,2030,FOM,2,%/year,Palzer thesis
-decentral gas boiler,2030,efficiency,0.9,per unit,Palzer thesis
-decentral gas boiler,2030,discount rate,0.04,per unit,Palzer thesis
-central gas boiler,2030,investment,63,EUR/kWhth,Palzer thesis
-central gas boiler,2030,lifetime,22,years,Palzer thesis
-central gas boiler,2030,FOM,1,%/year,Palzer thesis
-central gas boiler,2030,efficiency,0.9,per unit,Palzer thesis
-decentral CHP,2030,lifetime,25,years,HP
-decentral CHP,2030,investment,1400,EUR/kWel,HP
-decentral CHP,2030,FOM,3,%/year,HP
-decentral CHP,2030,discount rate,0.04,per unit,Palzer thesis
-central CHP,2030,lifetime,25,years,HP
-central CHP,2030,investment,650,EUR/kWel,HP
-central CHP,2030,FOM,3,%/year,HP
-decentral solar thermal,2030,discount rate,0.04,per unit,Palzer thesis
-decentral solar thermal,2030,FOM,1.3,%/year,HP
-decentral solar thermal,2030,investment,270000,EUR/1000m2,HP
-decentral solar thermal,2030,lifetime,20,years,HP
-central solar thermal,2030,FOM,1.4,%/year,HP
-central solar thermal,2030,investment,140000,EUR/1000m2,HP
-central solar thermal,2030,lifetime,20,years,HP
-HVAC overhead,2030,investment,400,EUR/MW/km,Hagspiel
-HVAC overhead,2030,lifetime,40,years,Hagspiel
-HVAC overhead,2030,FOM,2,%/year,Hagspiel
-HVDC overhead,2030,investment,400,EUR/MW/km,Hagspiel
-HVDC overhead,2030,lifetime,40,years,Hagspiel
-HVDC overhead,2030,FOM,2,%/year,Hagspiel
-HVDC submarine,2030,investment,2000,EUR/MW/km,DTU report based on Fig 34 of https://ec.europa.eu/energy/sites/ener/files/documents/2014_nsog_report.pdf
-HVDC submarine,2030,lifetime,40,years,Hagspiel
-HVDC submarine,2030,FOM,2,%/year,Hagspiel
-HVDC inverter pair,2030,investment,150000,EUR/MW,Hagspiel
-HVDC inverter pair,2030,lifetime,40,years,Hagspiel
-HVDC inverter pair,2030,FOM,2,%/year,Hagspiel

data/nuclear_p_max_pu.csv

@@ -1,16 +1,16 @@
 country,factor
-BE,0.65
-BG,0.89
-CZ,0.82
-FI,0.92
-FR,0.70
-DE,0.88
-HU,0.90
-NL,0.86
-RO,0.92
-SK,0.89
-SI,0.94
-ES,0.89
-SE,0.82
-CH,0.86
-GB,0.67
+BE,0.796
+BG,0.894
+CZ,0.827
+FI,0.936
+FR,0.71
+DE,0.871
+HU,0.913
+NL,0.868
+RO,0.909
+SK,0.9
+SI,0.913
+ES,0.897
+SE,0.851
+CH,0.87
+GB,0.656

data/override_component_attrs/buses.csv

@@ -1,3 +0,0 @@
-attribute,type,unit,default,description,status
-location,string,n/a,n/a,Reference to original electricity bus,Input (optional)
-unit,string,n/a,MWh,Unit of the bus (descriptive only), Input (optional)

data/override_component_attrs/generators.csv

@@ -1,4 +0,0 @@
-attribute,type,unit,default,description,status
-carrier,string,n/a,n/a,carrier,Input (optional)
-lifetime,float,years,inf,lifetime,Input (optional)
-build_year,int,year ,0,build year,Input (optional)

data/override_component_attrs/links.csv

@@ -1,13 +0,0 @@
-attribute,type,unit,default,description,status
-bus2,string,n/a,n/a,2nd bus,Input (optional)
-bus3,string,n/a,n/a,3rd bus,Input (optional)
-bus4,string,n/a,n/a,4th bus,Input (optional)
-efficiency2,static or series,per unit,1,2nd bus efficiency,Input (optional)
-efficiency3,static or series,per unit,1,3rd bus efficiency,Input (optional)
-efficiency4,static or series,per unit,1,4th bus efficiency,Input (optional)
-p2,series,MW,0,2nd bus output,Output
-p3,series,MW,0,3rd bus output,Output
-p4,series,MW,0,4th bus output,Output
-carrier,string,n/a,n/a,carrier,Input (optional)
-lifetime,float,years,inf,lifetime,Input (optional)
-build_year,int,year ,0,build year,Input (optional)

data/override_component_attrs/loads.csv

@@ -1,2 +0,0 @@
-attribute,type,unit,default,description,status
-carrier,string,n/a,n/a,carrier,Input (optional)

data/override_component_attrs/stores.csv

@@ -1,4 +0,0 @@
-attribute,type,unit,default,description,status
-carrier,string,n/a,n/a,carrier,Input (optional)
-lifetime,float,years,inf,lifetime,Input (optional)
-build_year,int,year ,0,build year,Input (optional)

data/unit_commitment.csv

@@ -0,0 +1,8 @@
+attribute,OCGT,CCGT,coal,lignite,nuclear
+ramp_limit_up,1,1,1,1,0.3
+ramp_limit_start_up,0.2,0.45,0.38,0.4,0.5
+ramp_limit_shut_down,0.2,0.45,0.38,0.4,0.5
+p_min_pu,0.2,0.45,0.325,0.4,0.5
+min_up_time,,3,5,7,6
+min_down_time,,2,6,6,10
+start_up_cost,9.6,34.2,35.64,19.14,16.5

doc/conf.py

@@ -82,7 +82,7 @@ author = "Tom Brown (KIT, TUB, FIAS), Jonas Hoersch (KIT, FIAS), Fabian Hofmann
 # The short X.Y version.
 version = "0.8"
 # The full version, including alpha/beta/rc tags.
-release = "0.8.0"
+release = "0.8.1"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

doc/configtables/clustering.csv

@@ -4,12 +4,13 @@ simplify_network,,,
 -- algorithm,str,"One of {‘kmeans’, ‘hac’, ‘modularity‘}",
 -- feature,str,"Str in the format ‘carrier1+carrier2+...+carrierN-X’, where CarrierI can be from {‘solar’, ‘onwind’, ‘offwind’, ‘ror’} and X is one of {‘cap’, ‘time’}.",
 -- exclude_carriers,list,"List of Str like [ 'solar', 'onwind'] or empy list []","List of carriers which will not be aggregated. If empty, all carriers will be aggregated."
--- remove stubs,bool,"true/false","Controls whether radial parts of the network should be recursively aggregated. Defaults to true."
--- remove_stubs_across_borders,bool,"true/false","Controls whether radial parts of the network should be recursively aggregated across borders. Defaults to true."
+-- remove stubs,bool,"{'true','false'}",Controls whether radial parts of the network should be recursively aggregated. Defaults to true.
+-- remove_stubs_across_borders,bool,"{'true','false'}",Controls whether radial parts of the network should be recursively aggregated across borders. Defaults to true.
 cluster_network,,,
 -- algorithm,str,"One of {‘kmeans’, ‘hac’}",
 -- feature,str,"Str in the format ‘carrier1+carrier2+...+carrierN-X’, where CarrierI can be from {‘solar’, ‘onwind’, ‘offwind’, ‘ror’} and X is one of {‘cap’, ‘time’}.",
 -- exclude_carriers,list,"List of Str like [ 'solar', 'onwind'] or empy list []","List of carriers which will not be aggregated. If empty, all carriers will be aggregated."
+-- consider_efficiency_classes,bool,"{'true','false'}","Aggregated each carriers into the top 10-quantile (high), the bottom 90-quantile (low), and everything in between (medium)."
 aggregation_strategies,,,
 -- generators,,,
 -- -- {key},str,"{key} can be any of the component of the generator (str). It’s value can be any that can be converted to pandas.Series using getattr(). For example one of {min, max, sum}.","Aggregates the component according to the given strategy. For example, if sum, then all values within each cluster are summed to represent the new generator."

doc/configtables/conventional.csv

@@ -1,3 +1,5 @@
 ,Unit,Values,Description
-{name},--,"string","For any carrier/technology overwrite attributes as listed below."
--- {attribute},--,"string or float","For any attribute, can specify a float or reference to a file path to a CSV file giving floats for each country (2-letter code)."
+unit_commitment ,bool,"{true, false}","Allow the overwrite of ramp_limit_up, ramp_limit_start_up, ramp_limit_shut_down, p_min_pu, min_up_time, min_down_time, and start_up_cost of conventional generators. Refer to the CSV file „unit_commitment.csv“."
+dynamic_fuel_price ,bool,"{true, false}","Consider the monthly fluctuating fuel prices for each conventional generator. Refer to the CSV file ""data/validation/monthly_fuel_price.csv""."
+{name},--,string,For any carrier/technology overwrite attributes as listed below.
+-- {attribute},--,string or float,"For any attribute, can specify a float or reference to a file path to a CSV file giving floats for each country (2-letter code)."

doc/configtables/hydro.csv

@@ -1,6 +1,8 @@
 ,Unit,Values,Description
-cutout,--,"Must be 'europe-2013-era5'","Specifies the directory where the relevant weather data ist stored."
+cutout,--,Must be 'europe-2013-era5',Specifies the directory where the relevant weather data ist stored.
 carriers,--,"Any subset of {'ror', 'PHS', 'hydro'}","Specifies the types of hydro power plants to build per-unit availability time series for. 'ror' stands for run-of-river plants, 'PHS' represents pumped-hydro storage, and 'hydro' stands for hydroelectric dams."
-PHS_max_hours,h,float,"Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation <https://pypsa.readthedocs.io/en/latest/components.html#storage-unit>`_."
-hydro_max_hours,h,"Any of {float, 'energy_capacity_totals_by_country', 'estimate_by_large_installations'}","Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom`` or heuristically determined. Cf. `PyPSA documentation <https://pypsa.readthedocs.io/en/latest/components.html#storage-unit>`_."
+PHS_max_hours,h,float,Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation <https://pypsa.readthedocs.io/en/latest/components.html#storage-unit>`_.
+hydro_max_hours,h,"Any of {float, 'energy_capacity_totals_by_country', 'estimate_by_large_installations'}",Maximum state of charge capacity of the pumped-hydro storage (PHS) in terms of hours at full output capacity ``p_nom`` or heuristically determined. Cf. `PyPSA documentation <https://pypsa.readthedocs.io/en/latest/components.html#storage-unit>`_.
+flatten_dispatch,bool,"{true, false}",Consider an upper limit for the hydro dispatch. The limit is given by the average capacity factor plus the buffer given in  ``flatten_dispatch_buffer``
+flatten_dispatch_buffer,--,float,"If ``flatten_dispatch`` is true, specify the value added above the average capacity factor."
 clip_min_inflow,MW,float,"To avoid too small values in the inflow time series, values below this threshold are set to zero."

doc/configtables/lines.csv

@@ -5,3 +5,9 @@ s_nom_max,MW,"float","Global upper limit for the maximum capacity of each extend
 max_extension,MW,"float","Upper limit for the extended capacity of each extendable line."
 length_factor,--,float,"Correction factor to account for the fact that buses are *not* connected by lines through air-line distance."
 under_construction,--,"One of {'zero': set capacity to zero, 'remove': remove completely, 'keep': keep with full capacity}","Specifies how to handle lines which are currently under construction."
+dynamic_line_rating,,,
+-- activate,bool,"true or false","Whether to take dynamic line rating into account"
+-- cutout,--,"Should be a folder listed in the configuration ``atlite: cutouts:`` (e.g. 'europe-2013-era5') or reference an existing folder in the directory ``cutouts``. Source module must be ERA5.","Specifies the directory where the relevant weather data ist stored."
+-- correction_factor,--,"float","Factor to compensate for overestimation of wind speeds in hourly averaged wind data"
+-- max_voltage_difference,deg,"float","Maximum voltage angle difference in degrees or 'false' to disable"
+-- max_line_rating,--,"float","Maximum line rating relative to nominal capacity without DLR, e.g. 1.3 or 'false' to disable"

doc/configtables/opts.csv

@@ -3,6 +3,7 @@ Trigger, Description, Definition, Status
 ``nSEG``; e.g. ``4380SEG``, "Apply time series segmentation with `tsam <https://tsam.readthedocs.io/en/latest/index.html>`_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load.", ``prepare_network``: apply_time_segmentation(), In active use
 ``Co2L``, Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``. If a float is appended an overall emission limit relative to the emission level given in ``electricity: co2base`` is added (e.g. ``Co2L0.05`` limits emissisions to 5% of what is given in ``electricity: co2base``), ``prepare_network``: `add_co2limit() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L19>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L154>`__, In active use
 ``Ep``, Add cost for a carbon-dioxide price configured in ``costs: emission_prices: co2`` to ``marginal_cost`` of generators (other emission types listed in ``network.carriers`` possible as well), ``prepare_network``: `add_emission_prices() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L24>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L158>`__, In active use
+``Ept``, Add monthly cost for a carbon-dioxide price based on historical values built by the rule ``build_monthly_prices``, In active use
 ``CCL``, Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``., ``solve_network``, In active use
 ``EQ``, "Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption.", ``solve_network``, In active use
 ``ATK``, "Require each node to be autarkic. Example: ``ATK`` removes all lines and links. ``ATKc`` removes all cross-border lines and links.", ``prepare_network``, In active use

doc/configtables/solving.csv

@@ -1,17 +1,19 @@
 ,Unit,Values,Description
 options,,,
+-- clip_p_max_pu,p.u.,float,To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero.
 -- load_shedding,bool/float,"{'true','false', float}","Add generators with very high marginal cost to simulate load shedding and avoid problem infeasibilities. If load shedding is a float, it denotes the marginal cost in EUR/kWh."
--- transmission_losses,int,"[0-9]","Add piecewise linear approximation of transmission losses based on n tangents. Defaults to 0, which means losses are ignored."
 -- noisy_costs,bool,"{'true','false'}","Add random noise to marginal cost of generators by :math:`\mathcal{U}(0.009,0,011)` and capital cost of lines and links by :math:`\mathcal{U}(0.09,0,11)`."
--- min_iterations,--,int,"Minimum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run."
--- max_iterations,--,int,"Maximum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run."
--- nhours,--,int,"Specifies the :math:`n` first snapshots to take into account. Must be less than the total number of snapshots. Rather recommended only for debugging."
--- clip_p_max_pu,p.u.,float,"To avoid too small values in the renewables` per-unit availability time series values below this threshold are set to zero."
 -- skip_iterations,bool,"{'true','false'}","Skip iterating, do not update impedances of branches. Defaults to true."
--- track_iterations,bool,"{'true','false'}","Flag whether to store the intermediate branch capacities and objective function values are recorded for each iteration in ``network.lines['s_nom_opt_X']`` (where ``X`` labels the iteration)"
--- seed,--,int,"Random seed for increased deterministic behaviour."
+-- rolling_horizon,bool,"{'true','false'}","Whether to optimize the network in a rolling horizon manner, where the snapshot range is split into slices of size `horizon` which are solved consecutively."
+-- seed,--,int,Random seed for increased deterministic behaviour.
+-- track_iterations,bool,"{'true','false'}",Flag whether to store the intermediate branch capacities and objective function values are recorded for each iteration in ``network.lines['s_nom_opt_X']`` (where ``X`` labels the iteration)
+-- min_iterations,--,int,Minimum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run.
+-- max_iterations,--,int,Maximum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run.
+-- transmission_losses,int,[0-9],"Add piecewise linear approximation of transmission losses based on n tangents. Defaults to 0, which means losses are ignored."
+-- linearized_unit_commitment,bool,"{'true','false'}",Whether to optimise using the linearized unit commitment formulation.
+-- horizon,--,int,Number of snapshots to consider in each iteration. Defaults to 100.
 solver,,,
--- name,--,"One of {'gurobi', 'cplex', 'cbc', 'glpk', 'ipopt'}; potentially more possible","Solver to use for optimisation problems in the workflow; e.g. clustering and linear optimal power flow."
--- options,--,"Key listed under ``solver_options``.","Link to specific parameter settings."
-solver_options,,"dict","Dictionaries with solver-specific parameter settings."
-mem,MB,"int","Estimated maximum memory requirement for solving networks."
+-- name,--,"One of {'gurobi', 'cplex', 'cbc', 'glpk', 'ipopt'}; potentially more possible",Solver to use for optimisation problems in the workflow; e.g. clustering and linear optimal power flow.
+-- options,--,Key listed under ``solver_options``.,Link to specific parameter settings.
+solver_options,,dict,Dictionaries with solver-specific parameter settings.
+mem,MB,int,Estimated maximum memory requirement for solving networks.

doc/configtables/toplevel.csv

@@ -1,6 +1,12 @@
 ,Unit,Values,Description
-version,--,0.x.x,"Version of PyPSA-Eur. Descriptive only."
-tutorial,bool,"{true, false}","Switch to retrieve the tutorial data set instead of the full data set."
+version,--,0.x.x,Version of PyPSA-Eur. Descriptive only.
+tutorial,bool,"{true, false}",Switch to retrieve the tutorial data set instead of the full data set.
 logging,,,
 -- level,--,"Any of {'INFO', 'WARNING', 'ERROR'}","Restrict console outputs to all infos, warning or errors only"
--- format,--,"","Custom format for log messages. See `LogRecord <https://docs.python.org/3/library/logging.html#logging.LogRecord>`_ attributes."
+-- format,--,,Custom format for log messages. See `LogRecord <https://docs.python.org/3/library/logging.html#logging.LogRecord>`_ attributes.
+private,,,
+-- keys,,,
+-- -- entsoe_api,--,,Optionally specify the ENTSO-E API key. See the guidelines to get `ENTSO-E API key <https://transparency.entsoe.eu/content/static_content/Static%20content/web%20api/Guide.html>`_
+remote,,,
+-- ssh,--,,Optionally specify the SSH of a remote cluster to be synchronized.
+-- path,--,,Optionally specify the file path within the remote cluster to be synchronized.

doc/configuration.rst

@@ -16,12 +16,13 @@ PyPSA-Eur has several configuration options which are documented in this section
 Top-level configuration
 =======================
 
+"Private" refers to local, machine-specific settings or data meant for personal use, not to be shared. "Remote" indicates the address of a server used for data exchange, often for clusters and data pushing/pulling.
+
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: version:
    :end-before: # docs
 
-
 .. csv-table::
    :header-rows: 1
    :widths: 22,7,22,33

doc/index.rst

@@ -78,10 +78,10 @@ them:
 
 .. note::
     You can find showcases of the model's capabilities in the Supplementary Materials of the
-    preprint `Benefits of a Hydrogen Network in Europe
-    <https://arxiv.org/abs/2207.05816>`_, the Supplementary Materials of the `paper in Joule with a
+    Joule paper `The potential role of a hydrogen network in Europe
+    <https://doi.org/10.1016/j.joule.2023.06.016>`_, the Supplementary Materials of another `paper in Joule with a
     description of the industry sector
-    <https://arxiv.org/abs/2109.09563>`_, or in `a 2021 presentation
+    <https://doi.org/10.1016/j.joule.2022.04.016>`_, or in `a 2021 presentation
     at EMP-E <https://nworbmot.org/energy/brown-empe.pdf>`_.
     The sector-coupled extension of PyPSA-Eur was
     initially described in the paper `Synergies of sector coupling and transmission
@@ -179,10 +179,13 @@ For sector-coupling studies: ::
 
     @misc{PyPSAEurSec,
         author = "Fabian Neumann and Elisabeth Zeyen and Marta Victoria and Tom Brown",
-        title = "The Potential Role of a Hydrogen Network in Europe",
-        year = "2022",
+        title = "The potential role of a hydrogen network in Europe",
+        journal "Joule",
+        volume = "7",
+        pages = "1--25"
+        year = "2023",
         eprint = "2207.05816",
-        url = "https://arxiv.org/abs/2207.05816",
+        doi = "10.1016/j.joule.2022.04.016",
     }
 
 For sector-coupling studies with pathway optimisation: ::
@@ -224,7 +227,10 @@ The included ``.nc`` files are PyPSA network files which can be imported with Py
     n = pypsa.Network(filename)
 
 
+Operating Systems
+=================
 
+The PyPSA-Eur workflow is continuously tested for Linux, macOS and Windows (WSL only).
 
 
 .. toctree::
@@ -274,6 +280,7 @@ The included ``.nc`` files are PyPSA network files which can be imported with Py
 
    release_notes
    licenses
+   validation
    limitations
    contributing
    support

doc/release_notes.rst

@@ -10,39 +10,131 @@ Release Notes
 Upcoming Release
 ================
 
-* ``param:`` section in rule definition are added to track changed settings in ``config.yaml``. The goal is to automatically re-execute rules whose parameters have changed. See `Non-file parameters for rules <https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#non-file-parameters-for-rules>`_ in the snakemake documentation.
+* Updated Global Energy Monitor LNG terminal data to March 2023 version.
 
-* **Important:** The configuration files are now located in the ``config`` directory. This counts for ``config.default.yaml``, ``config.yaml`` as well as the test configuration files which are now located in ``config/test``. Config files that are still in the root directory will be ignored.
+* For industry distribution, use EPRTR as fallback if ETS data is not available.
 
-* Bugfix: Correct typo in the CPLEX solver configuration in ``config.default.yaml``.
+PyPSA-Eur 0.8.1 (27th July 2023)
+================================
 
-* Bugfix: Error in ``add_electricity`` where carriers were added multiple times to the network, resulting in a non-unique carriers error.
+**New Features**
 
-* Renamed script file from PyPSA-EUR ``build_load_data`` to ``build_electricity_demand`` and ``retrieve_load_data`` to ``retrieve_electricity_demand``.
+* Add option to consider dynamic line rating based on wind speeds and
+  temperature according to `Glaum and Hofmann (2022)
+  <https://arxiv.org/abs/2208.04716>`_. See configuration section ``lines:
+  dynamic_line_rating:`` for more details. (https://github.com/PyPSA/pypsa-eur/pull/675)
 
-* Fix docs readthedocs built
+* Add option to include a piecewise linear approximation of transmission losses,
+  e.g. by setting ``solving: options: transmission_losses: 2`` for an
+  approximation with two tangents. (https://github.com/PyPSA/pypsa-eur/pull/664)
 
 * Add plain hydrogen turbine as additional re-electrification option besides
   hydrogen fuel cell. Add switches for both re-electrification options under
   ``sector: hydrogen_turbine:`` and ``sector: hydrogen_fuel_cell:``.
-
-* A new function named ``sanitize_carrier`` ensures that all unique carrier names are present in the network's carriers attribute, and adds nice names and colors for each carrier according to the provided configuration dictionary.
-
-* Additional tech_color are added to include previously unlisted carriers.
-
-* Remove ``vresutils`` dependency.
+  (https://github.com/PyPSA/pypsa-eur/pull/647)
 
 * Added configuration option ``lines: max_extension:`` and ``links:
   max_extension:``` to control the maximum capacity addition per line or link in
-  MW.
+  MW. (https://github.com/PyPSA/pypsa-eur/pull/665)
 
-* Add option to include a piecewise linear approximation of transmission losses,
-  e.g. by setting ``solving: options: transmission_losses: 2`` for an
-  approximation with two tangents.
+* A ``param:`` section in the snakemake rule definitions was added to track
+  changed settings in ``config.yaml``. The goal is to automatically re-execute
+  rules where parameters have changed. See `Non-file parameters for rules
+  <https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#non-file-parameters-for-rules>`_
+  in the snakemake documentation. (https://github.com/PyPSA/pypsa-eur/pull/663)
+
+* A new function named ``sanitize_carrier`` ensures that all unique carrier
+  names are present in the network's carriers attribute, and adds nice names and
+  colors for each carrier according to the provided configuration dictionary.
+  (https://github.com/PyPSA/pypsa-eur/pull/653,
+  https://github.com/PyPSA/pypsa-eur/pull/690)
+
+* The configuration settings have been documented in more detail.
+  (https://github.com/PyPSA/pypsa-eur/pull/685)
+
+**Breaking Changes**
+
+* The configuration files are now located in the ``config`` directory. This
+  includes the ``config.default.yaml``, ``config.yaml`` as well as the test
+  configuration files which are now located in the ``config/test`` directory.
+  Config files that are still in the root directory will be ignored.
+  (https://github.com/PyPSA/pypsa-eur/pull/640)
+
+* Renamed script and rule name from ``build_load_data`` to
+  ``build_electricity_demand`` and ``retrieve_load_data`` to
+  ``retrieve_electricity_demand``. (https://github.com/PyPSA/pypsa-eur/pull/642,
+  https://github.com/PyPSA/pypsa-eur/pull/652)
+
+* Updated to new spatial clustering module introduced in PyPSA v0.25.
+  (https://github.com/PyPSA/pypsa-eur/pull/696)
+
+**Changes**
+
+* Handling networks with links with multiple inputs/outputs no longer requires
+  to override component attributes.
+  (https://github.com/PyPSA/pypsa-eur/pull/695)
 
 * Added configuration option ``enable: retrieve:`` to control whether data
   retrieval rules from snakemake are enabled or not. Th default setting ``auto``
-  will automatically detect and enable/disable the rules based on internet connectivity.
+  will automatically detect and enable/disable the rules based on internet
+  connectivity. (https://github.com/PyPSA/pypsa-eur/pull/694)
+
+* Update to ``technology-data`` v0.6.0.
+  (https://github.com/PyPSA/pypsa-eur/pull/704)
+
+* Handle data bundle extraction paths via ``snakemake.output``.
+
+* Additional technologies are added to ``tech_color`` in the configuration files
+  to include previously unlisted carriers.
+
+* Doc: Added note that Windows is only tested in CI with WSL.
+  (https://github.com/PyPSA/pypsa-eur/issues/697)
+
+* Doc: Add support section. (https://github.com/PyPSA/pypsa-eur/pull/656)
+
+* Open ``rasterio`` files with ``rioxarray``.
+  (https://github.com/PyPSA/pypsa-eur/pull/474)
+
+* Migrate CI to ``micromamba``. (https://github.com/PyPSA/pypsa-eur/pull/700)
+
+**Bugs and Compatibility**
+
+* The new minimum PyPSA version is v0.25.1.
+
+* Removed ``vresutils`` dependency.
+  (https://github.com/PyPSA/pypsa-eur/pull/662)
+
+* Adapt to new ``powerplantmatching`` version.
+  (https://github.com/PyPSA/pypsa-eur/pull/687,
+  https://github.com/PyPSA/pypsa-eur/pull/701)
+
+* Bugfix: Correct typo in the CPLEX solver configuration in
+  ``config.default.yaml``. (https://github.com/PyPSA/pypsa-eur/pull/630)
+
+* Bugfix: Error in ``add_electricity`` where carriers were added multiple times
+  to the network, resulting in a non-unique carriers error.
+
+* Bugfix of optional reserve constraint.
+  (https://github.com/PyPSA/pypsa-eur/pull/645)
+
+* Fix broken equity constraints logic.
+  (https://github.com/PyPSA/pypsa-eur/pull/679)
+
+* Fix addition of load shedding generators.
+  (https://github.com/PyPSA/pypsa-eur/pull/649)
+
+* Fix automatic building of documentation on readthedocs.org.
+  (https://github.com/PyPSA/pypsa-eur/pull/658)
+
+* Bugfix: Update network clustering to avoid adding deleted links in clustered
+  network. (https://github.com/PyPSA/pypsa-eur/pull/678)
+
+* Address ``geopandas`` deprecations.
+  (https://github.com/PyPSA/pypsa-eur/pull/678)
+
+* Fix bug with underground hydrogen storage creation, where for some small model
+  regions no cavern storage is available.
+  (https://github.com/PyPSA/pypsa-eur/pull/672)
 
 
 PyPSA-Eur 0.8.0 (18th March 2023)

doc/retrieve.rst

@@ -83,7 +83,7 @@ This rule, as a substitute for :mod:`build_natura_raster`, downloads an already
 Rule ``retrieve_electricity_demand``
 ====================================
 
-This rule downloads hourly electric load data for each country from the `OPSD platform <data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv>`_.
+This rule downloads hourly electric load data for each country from the `OPSD platform <https://data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv>`_.
 
 **Relevant Settings**
 

doc/validation.rst

@@ -0,0 +1,53 @@
+..
+  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
+
+  SPDX-License-Identifier: CC-BY-4.0
+
+##########################################
+Validation
+##########################################
+
+The PyPSA-Eur model workflow provides a built-in mechanism for validation. This allows users to contrast the outcomes of network optimization against the historical behaviour of the European power system. The snakemake rule ``validate_elec_networks`` enables this by generating comparative figures that encapsulate key data points such as dispatch carrier, cross-border flows, and market prices per price zone.
+
+These comparisons utilize data from the 2019 ENTSO-E Transparency Platform. To enable this, an ENTSO-E API key must be inserted into the ``config.yaml`` file. Detailed steps for this process can be found in the user guide `here <https://transparency.entsoe.eu/content/static_content/Static%20content/web%20api/Guide.html>`_.
+
+Once the API key is set, the validation workflow can be triggered by running the following command:
+
+    snakemake validate_elec_networks --configfile config/config.validation.yaml -c8
+
+
+The configuration file `config/config.validation.yaml` contains the following parameters:
+
+.. literalinclude:: ../config/config.validation.yaml
+   :language: yaml
+
+The setup uses monthly varying fuel prices for gas, lignite, coal and oil as well as CO2 prices, which are created by the script ``build_monthly_prices``. Upon completion of the validation process, the resulting network and generated figures will be stored in the ``results/validation`` directory for further analysis.
+
+
+Results
+=======
+
+By the time of writing the comparison with the historical data shows partially accurate, partially improvable results. The following figures show the comparison of the dispatch of the different carriers.
+
+.. image:: ../graphics/validation_seasonal_operation_area_elec_s_37_ec_lv1.0_Ept.png
+   :width: 100%
+   :align: center
+
+.. image:: ../graphics/validation_production_bar_elec_s_37_ec_lv1.0_Ept.png
+   :width: 100%
+   :align: center
+
+
+
+Issues and possible improvements
+--------------------------------
+
+**Overestimated dispatch of wind and solar:** Renewable potentials of wind and solar are slightly overestimated in the model. This leads to a higher dispatch of these carriers than in the historical data. In particular, the solar dispatch during winter is overestimated.
+
+**Coal - Lignite fuel switch:** The model has a fuel switch from coal to lignite. This might result from non-captured subsidies for lignite and coal in the model. In order to fix the fuel switch from coal to lignite, a manual cost correction was added to the script ``build_monthly_prices``.
+
+**Planned outages of nuclear power plants:** Planned outages of nuclear power plants are not captured in the model. This leads to a underestimated dispatch of nuclear power plants in winter and a overestimated dispatch in summer. This point is hard to fix, since the planned outages are not published in the ENTSO-E Transparency Platform.
+
+**False classification of run-of-river power plants:** Some run-of-river power plants are classified as hydro power plants in the model. This leads to a general overestimation of the hydro power dispatch. In particular, Swedish hydro power plants are overestimated.
+
+**Load shedding:** Due to constraint NTC's (crossborder capacities), the model has to shed load in some regions. This leads to a high market prices in the regions which drive the average market price up. Further fine-tuning of the NTC's is needed to avoid load shedding.

envs/environment.fixed.yaml

@@ -12,74 +12,93 @@ dependencies:
 - _libgcc_mutex=0.1
 - _openmp_mutex=4.5
 - affine=2.4.0
-- alsa-lib=1.2.8
+- alsa-lib=1.2.9
 - ampl-mp=3.1.0
-- amply=0.1.5
+- amply=0.1.6
+- anyio=3.7.1
 - appdirs=1.4.4
+- argon2-cffi=21.3.0
+- argon2-cffi-bindings=21.2.0
 - asttokens=2.2.1
-- atlite=0.2.10
+- async-lru=2.0.3
+- atk-1.0=2.38.0
+- atlite=0.2.11
 - attr=2.5.1
-- attrs=22.2.0
+- attrs=23.1.0
+- aws-c-auth=0.7.0
+- aws-c-cal=0.6.0
+- aws-c-common=0.8.23
+- aws-c-compression=0.2.17
+- aws-c-event-stream=0.3.1
+- aws-c-http=0.7.11
+- aws-c-io=0.13.28
+- aws-c-mqtt=0.8.14
+- aws-c-s3=0.3.13
+- aws-c-sdkutils=0.1.11
+- aws-checksums=0.1.16
+- aws-crt-cpp=0.20.3
+- aws-sdk-cpp=1.10.57
+- babel=2.12.1
 - backcall=0.2.0
 - backports=1.0
-- backports.functools_lru_cache=1.6.4
-- beautifulsoup4=4.11.2
-- blosc=1.21.3
-- bokeh=2.4.3
+- backports.functools_lru_cache=1.6.5
+- beautifulsoup4=4.12.2
+- bleach=6.0.0
+- blosc=1.21.4
+- bokeh=3.2.1
 - boost-cpp=1.78.0
-- bottleneck=1.3.6
+- bottleneck=1.3.7
 - branca=0.6.0
 - brotli=1.0.9
 - brotli-bin=1.0.9
-- brotlipy=0.7.0
+- brotli-python=1.0.9
 - bzip2=1.0.8
-- c-ares=1.18.1
-- ca-certificates=2022.12.7
+- c-ares=1.19.1
+- c-blosc2=2.10.0
+- ca-certificates=2023.7.22
 - cairo=1.16.0
 - cartopy=0.21.1
-- cdsapi=0.5.1
-- certifi=2022.12.7
+- cdsapi=0.6.1
+- certifi=2023.7.22
 - cffi=1.15.1
 - cfitsio=4.2.0
 - cftime=1.6.2
-- charset-normalizer=2.1.1
-- click=8.1.3
+- charset-normalizer=3.2.0
+- click=8.1.6
 - click-plugins=1.1.1
 - cligj=0.7.2
 - cloudpickle=2.2.1
-- coin-or-cbc=2.10.8
-- coin-or-cgl=0.60.6
-- coin-or-clp=1.17.7
-- coin-or-osi=0.108.7
-- coin-or-utils=2.11.6
-- coincbc=2.10.8
 - colorama=0.4.6
-- configargparse=1.5.3
+- comm=0.1.3
+- configargparse=1.7
 - connection_pool=0.0.3
-- country_converter=0.8.0
-- cryptography=39.0.1
-- curl=7.88.0
+- contourpy=1.1.0
+- country_converter=1.0.0
+- curl=8.2.0
 - cycler=0.11.0
-- cytoolz=0.12.0
-- dask=2023.2.0
-- dask-core=2023.2.0
+- cytoolz=0.12.2
+- dask=2023.7.1
+- dask-core=2023.7.1
 - datrie=0.8.2
 - dbus=1.13.6
+- debugpy=1.6.7
 - decorator=5.1.1
+- defusedxml=0.7.1
 - deprecation=2.1.0
 - descartes=1.1.0
-- distributed=2023.2.0
+- distributed=2023.7.1
 - distro=1.8.0
-- docutils=0.19
-- dpath=2.1.4
-- entsoe-py=0.5.8
+- docutils=0.20.1
+- dpath=2.1.6
+- entrypoints=0.4
+- entsoe-py=0.5.10
 - et_xmlfile=1.1.0
-- exceptiongroup=1.1.0
+- exceptiongroup=1.1.2
 - executing=1.2.0
 - expat=2.5.0
-- fftw=3.3.10
-- filelock=3.9.0
-- fiona=1.9.1
+- filelock=3.12.2
+- fiona=1.9.4
+- flit-core=3.9.0
 - folium=0.14.0
 - font-ttf-dejavu-sans-mono=2.37
 - font-ttf-inconsolata=3.000
@@ -88,293 +107,366 @@ dependencies:
 - fontconfig=2.14.2
 - fonts-conda-ecosystem=1
 - fonts-conda-forge=1
-- fonttools=4.38.0
+- fonttools=4.41.1
 - freetype=2.12.1
 - freexl=1.0.6
-- fsspec=2023.1.0
-- gdal=3.6.2
+- fribidi=1.0.10
+- fsspec=2023.6.0
+- gdal=3.7.0
+- gdk-pixbuf=2.42.10
 - geographiclib=1.52
 - geojson-rewind=1.0.2
-- geopandas=0.12.2
-- geopandas-base=0.12.2
+- geopandas=0.13.2
+- geopandas-base=0.13.2
 - geopy=2.3.0
-- geos=3.11.1
+- geos=3.11.2
 - geotiff=1.7.1
 - gettext=0.21.1
+- gflags=2.2.2
 - giflib=5.2.1
 - gitdb=4.0.10
-- gitpython=3.1.30
-- glib=2.74.1
-- glib-tools=2.74.1
+- gitpython=3.1.32
+- glib=2.76.4
+- glib-tools=2.76.4
+- glog=0.6.0
+- gmp=6.2.1
 - graphite2=1.3.13
-- gst-plugins-base=1.22.0
-- gstreamer=1.22.0
-- gstreamer-orc=0.4.33
-- harfbuzz=6.0.0
+- graphviz=8.1.0
+- gst-plugins-base=1.22.5
+- gstreamer=1.22.5
+- gtk2=2.24.33
+- gts=0.7.6
+- harfbuzz=7.3.0
 - hdf4=4.2.15
-- hdf5=1.12.2
-- heapdict=1.0.1
+- hdf5=1.14.1
 - humanfriendly=10.0
-- icu=70.1
+- icu=72.1
 - idna=3.4
-- importlib-metadata=6.0.0
-- importlib_resources=5.10.2
+- importlib-metadata=6.8.0
+- importlib_metadata=6.8.0
+- importlib_resources=6.0.0
 - iniconfig=2.0.0
-- ipopt=3.14.11
-- ipython=8.10.0
-- jack=1.9.22
+- ipopt=3.14.12
+- ipykernel=6.24.0
+- ipython=8.14.0
+- ipython_genutils=0.2.0
+- ipywidgets=8.0.7
 - jedi=0.18.2
 - jinja2=3.1.2
-- joblib=1.2.0
-- jpeg=9e
+- joblib=1.3.0
 - json-c=0.16
-- jsonschema=4.17.3
-- jupyter_core=5.2.0
-- kealib=1.5.0
+- json5=0.9.14
+- jsonschema=4.18.4
+- jsonschema-specifications=2023.7.1
+- jupyter=1.0.0
+- jupyter-lsp=2.2.0
+- jupyter_client=8.3.0
+- jupyter_console=6.6.3
+- jupyter_core=5.3.1
+- jupyter_events=0.6.3
+- jupyter_server=2.7.0
+- jupyter_server_terminals=0.4.4
+- jupyterlab=4.0.3
+- jupyterlab_pygments=0.2.2
+- jupyterlab_server=2.24.0
+- jupyterlab_widgets=3.0.8
+- kealib=1.5.1
 - keyutils=1.6.1
 - kiwisolver=1.4.4
-- krb5=1.20.1
+- krb5=1.21.1
 - lame=3.100
-- lcms2=2.14
+- lcms2=2.15
 - ld_impl_linux-64=2.40
 - lerc=4.0.0
+- libabseil=20230125.3
 - libaec=1.0.6
+- libarchive=3.6.2
+- libarrow=12.0.1
 - libblas=3.9.0
 - libbrotlicommon=1.0.9
 - libbrotlidec=1.0.9
 - libbrotlienc=1.0.9
-- libcap=2.66
+- libcap=2.67
 - libcblas=3.9.0
 - libclang=15.0.7
 - libclang13=15.0.7
+- libcrc32c=1.1.2
 - libcups=2.3.3
-- libcurl=7.88.0
-- libdb=6.2.32
-- libdeflate=1.17
+- libcurl=8.2.0
+- libdeflate=1.18
 - libedit=3.1.20191231
 - libev=4.33
-- libevent=2.1.10
+- libevent=2.1.12
+- libexpat=2.5.0
 - libffi=3.4.2
-- libflac=1.4.2
-- libgcc-ng=12.2.0
+- libflac=1.4.3
+- libgcc-ng=13.1.0
 - libgcrypt=1.10.1
-- libgdal=3.6.2
-- libgfortran-ng=12.2.0
-- libgfortran5=12.2.0
-- libglib=2.74.1
-- libgomp=12.2.0
-- libgpg-error=1.46
+- libgd=2.3.3
+- libgdal=3.7.0
+- libgfortran-ng=13.1.0
+- libgfortran5=13.1.0
+- libglib=2.76.4
+- libgomp=13.1.0
+- libgoogle-cloud=2.12.0
+- libgpg-error=1.47
+- libgrpc=1.56.2
 - libiconv=1.17
+- libjpeg-turbo=2.1.5.1
 - libkml=1.3.0
 - liblapack=3.9.0
 - liblapacke=3.9.0
 - libllvm15=15.0.7
-- libnetcdf=4.8.1
-- libnghttp2=1.51.0
+- libnetcdf=4.9.2
+- libnghttp2=1.52.0
 - libnsl=2.0.0
+- libnuma=2.0.16
 - libogg=1.3.4
-- libopenblas=0.3.21
+- libopenblas=0.3.23
 - libopus=1.3.1
 - libpng=1.6.39
-- libpq=15.2
+- libpq=15.3
+- libprotobuf=4.23.3
+- librsvg=2.56.1
 - librttopo=1.1.0
 - libsndfile=1.2.0
+- libsodium=1.0.18
 - libspatialindex=1.9.3
 - libspatialite=5.0.1
-- libsqlite=3.40.0
-- libssh2=1.10.0
-- libstdcxx-ng=12.2.0
-- libsystemd0=252
-- libtiff=4.5.0
+- libsqlite=3.42.0
+- libssh2=1.11.0
+- libstdcxx-ng=13.1.0
+- libsystemd0=253
+- libthrift=0.18.1
+- libtiff=4.5.1
 - libtool=2.4.7
-- libudev1=252
-- libuuid=2.32.1
+- libutf8proc=2.8.0
+- libuuid=2.38.1
 - libvorbis=1.3.7
-- libwebp-base=1.2.4
-- libxcb=1.13
+- libwebp=1.3.1
+- libwebp-base=1.3.1
+- libxcb=1.15
 - libxkbcommon=1.5.0
-- libxml2=2.10.3
+- libxml2=2.11.4
 - libxslt=1.1.37
 - libzip=1.9.2
 - libzlib=1.2.13
-- linopy=0.1.3
 - locket=1.0.0
-- lxml=4.9.2
+- lxml=4.9.3
 - lz4=4.3.2
 - lz4-c=1.9.4
 - lzo=2.10
 - mapclassify=2.5.0
-- markupsafe=2.1.2
+- markupsafe=2.1.3
 - matplotlib=3.5.3
 - matplotlib-base=3.5.3
 - matplotlib-inline=0.1.6
 - memory_profiler=0.61.0
-- metis=5.1.0
-- mpg123=1.31.2
-- msgpack-python=1.0.4
+- metis=5.1.1
+- mistune=3.0.0
+- mpg123=1.31.3
+- msgpack-python=1.0.5
 - mumps-include=5.2.1
 - mumps-seq=5.2.1
-- munch=2.5.0
+- munch=4.0.0
 - munkres=1.1.4
-- mysql-common=8.0.32
-- mysql-libs=8.0.32
-- nbformat=5.7.3
-- ncurses=6.3
-- netcdf4=1.6.2
-- networkx=3.0
+- mysql-common=8.0.33
+- mysql-libs=8.0.33
+- nbclient=0.8.0
+- nbconvert=7.7.2
+- nbconvert-core=7.7.2
+- nbconvert-pandoc=7.7.2
+- nbformat=5.9.1
+- ncurses=6.4
+- nest-asyncio=1.5.6
+- netcdf4=1.6.4
+- networkx=3.1
 - nomkl=1.0
+- notebook=7.0.0
+- notebook-shim=0.2.3
 - nspr=4.35
-- nss=3.88
-- numexpr=2.8.3
-- numpy=1.24
+- nss=3.89
+- numexpr=2.8.4
+- numpy=1.25.1
 - openjdk=17.0.3
 - openjpeg=2.5.0
-- openpyxl=3.1.0
-- openssl=3.0.8
-- packaging=23.0
-- pandas=1.5.3
+- openpyxl=3.1.2
+- openssl=3.1.1
+- orc=1.9.0
+- overrides=7.3.1
+- packaging=23.1
+- pandas=2.0.3
+- pandoc=3.1.3
+- pandocfilters=1.5.0
+- pango=1.50.14
 - parso=0.8.3
-- partd=1.3.0
+- partd=1.4.0
 - patsy=0.5.3
 - pcre2=10.40
 - pexpect=4.8.0
 - pickleshare=0.7.5
-- pillow=9.4.0
-- pip=23.0
+- pillow=10.0.0
+- pip=23.2.1
 - pixman=0.40.0
 - pkgutil-resolve-name=1.3.10
 - plac=1.3.5
-- platformdirs=3.0.0
-- pluggy=1.0.0
+- platformdirs=3.9.1
+- pluggy=1.2.0
 - ply=3.11
-- pooch=1.6.0
-- poppler=22.12.0
+- pooch=1.7.0
+- poppler=23.05.0
 - poppler-data=0.4.12
-- postgresql=15.2
-- powerplantmatching=0.5.6
+- postgresql=15.3
+- powerplantmatching=0.5.7
 - progressbar2=4.2.0
-- proj=9.1.0
-- prompt-toolkit=3.0.36
-- psutil=5.9.4
+- proj=9.2.1
+- prometheus_client=0.17.1
+- prompt-toolkit=3.0.39
+- prompt_toolkit=3.0.39
+- psutil=5.9.5
 - pthread-stubs=0.4
 - ptyprocess=0.7.0
 - pulp=2.7.0
-- pulseaudio=16.1
+- pulseaudio-client=16.1
 - pure_eval=0.2.2
+- py-cpuinfo=9.0.0
+- pyarrow=12.0.1
 - pycountry=22.3.5
 - pycparser=2.21
-- pygments=2.14.0
-- pyomo=6.4.4
-- pyopenssl=23.0.0
-- pyparsing=3.0.9
-- pyproj=3.4.1
-- pypsa=0.22.1
+- pygments=2.15.1
+- pyomo=6.6.1
+- pyparsing=3.1.0
+- pyproj=3.6.0
 - pyqt=5.15.7
 - pyqt5-sip=12.11.0
-- pyrsistent=0.19.3
 - pyshp=2.3.1
 - pysocks=1.7.1
-- pytables=3.7.0
-- pytest=7.2.1
-- python=3.10.9
+- pytables=3.8.0
+- pytest=7.4.0
+- python=3.10.12
 - python-dateutil=2.8.2
-- python-fastjsonschema=2.16.2
-- python-utils=3.5.2
+- python-fastjsonschema=2.18.0
+- python-json-logger=2.0.7
+- python-tzdata=2023.3
+- python-utils=3.7.0
 - python_abi=3.10
-- pytz=2022.7.1
+- pytz=2023.3
 - pyxlsb=1.0.10
 - pyyaml=6.0
+- pyzmq=25.1.0
 - qt-main=5.15.8
-- rasterio=1.3.4
-- readline=8.1.2
-- requests=2.28.1
-- retry=0.9.2
-- rich=12.5.1
-- rioxarray=0.13.3
-- rtree=1.0.0
-- s2n=1.0.10
-- scikit-learn=1.1.1
-- scipy=1.8.1
+- qtconsole=5.4.3
+- qtconsole-base=5.4.3
+- qtpy=2.3.1
+- rasterio=1.3.8
+- rdma-core=28.9
+- re2=2023.03.02
+- readline=8.2
+- referencing=0.30.0
+- requests=2.31.0
+- reretry=0.11.8
+- rfc3339-validator=0.1.4
+- rfc3986-validator=0.1.1
+- rioxarray=0.14.1
+- rpds-py=0.9.2
+- rtree=1.0.1
+- s2n=1.3.46
+- scikit-learn=1.3.0
+- scipy=1.11.1
 - scotch=6.0.9
 - seaborn=0.12.2
 - seaborn-base=0.12.2
-- setuptools=67.3.2
+- send2trash=1.8.2
+- setuptools=68.0.0
 - setuptools-scm=7.1.0
 - setuptools_scm=7.1.0
 - shapely=2.0.1
-- sip=6.7.7
+- sip=6.7.10
 - six=1.16.0
 - smart_open=6.3.0
 - smmap=3.0.5
-- snakemake-minimal=7.22.0
-- snappy=1.1.9
+- snakemake-minimal=7.30.2
+- snappy=1.1.10
+- sniffio=1.3.0
 - snuggs=1.4.7
 - sortedcontainers=2.4.0
 - soupsieve=2.3.2.post1
-- sqlite=3.40.0
+- sqlite=3.42.0
 - stack_data=0.6.2
-- statsmodels=0.13.5
+- statsmodels=0.14.0
 - stopit=1.1.2
 - tabula-py=2.6.0
 - tabulate=0.9.0
 - tblib=1.7.0
-- threadpoolctl=3.1.0
+- terminado=0.17.1
+- threadpoolctl=3.2.0
 - throttler=1.2.1
 - tiledb=2.13.2
+- tinycss2=1.2.1
 - tk=8.6.12
 - toml=0.10.2
 - tomli=2.0.1
 - toolz=0.12.0
-- toposort=1.9
-- tornado=6.2
-- tqdm=4.64.1
+- toposort=1.10
+- tornado=6.3.2
+- tqdm=4.65.0
 - traitlets=5.9.0
-- typing-extensions=4.4.0
-- typing_extensions=4.4.0
-- tzcode=2022g
-- tzdata=2022g
+- typing-extensions=4.7.1
+- typing_extensions=4.7.1
+- typing_utils=0.1.0
+- tzcode=2023c
+- tzdata=2023c
+- ucx=1.14.1
 - unicodedata2=15.0.0
 - unidecode=1.3.6
 - unixodbc=2.3.10
-- urllib3=1.26.14
+- urllib3=2.0.4
 - wcwidth=0.2.6
-- wheel=0.38.4
-- wrapt=1.14.1
-- xarray=2023.2.0
+- webencodings=0.5.1
+- websocket-client=1.6.1
+- wheel=0.41.0
+- widgetsnbextension=4.0.8
+- wrapt=1.15.0
+- xarray=2023.7.0
 - xcb-util=0.4.0
 - xcb-util-image=0.4.0
 - xcb-util-keysyms=0.4.0
 - xcb-util-renderutil=0.3.9
 - xcb-util-wm=0.4.1
 - xerces-c=3.2.4
+- xkeyboard-config=2.39
 - xlrd=2.0.1
 - xorg-fixesproto=5.0
 - xorg-inputproto=2.3.2
 - xorg-kbproto=1.0.7
-- xorg-libice=1.0.10
-- xorg-libsm=1.2.3
-- xorg-libx11=1.7.2
-- xorg-libxau=1.0.9
+- xorg-libice=1.1.1
+- xorg-libsm=1.2.4
+- xorg-libx11=1.8.6
+- xorg-libxau=1.0.11
 - xorg-libxdmcp=1.1.3
 - xorg-libxext=1.3.4
 - xorg-libxfixes=5.0.3
 - xorg-libxi=1.7.10
-- xorg-libxrender=0.9.10
+- xorg-libxrender=0.9.11
 - xorg-libxtst=1.2.3
 - xorg-recordproto=1.14.2
 - xorg-renderproto=0.11.1
 - xorg-xextproto=7.3.0
+- xorg-xf86vidmodeproto=2.3.1
 - xorg-xproto=7.0.31
-- xyzservices=2022.9.0
+- xyzservices=2023.7.0
 - xz=5.2.6
 - yaml=0.2.5
 - yte=1.5.1
-- zict=2.2.0
-- zipp=3.13.0
+- zeromq=4.3.4
+- zict=3.0.0
+- zipp=3.16.2
 - zlib=1.2.13
+- zlib-ng=2.0.7
 - zstd=1.5.2
 - pip:
-  - countrycode==0.2
-  - highspy==1.5.0.dev0
-  - pybind11==2.10.3
-  - tsam==2.2.2
+  - gurobipy==10.0.2
+  - linopy==0.2.2
+  - pypsa==0.25.1
+  - tsam==2.3.0
+  - validators==0.20.0

envs/environment.yaml

@@ -10,7 +10,6 @@ dependencies:
 - python>=3.8
 - pip
 
-# - pypsa>=0.23
 - atlite>=0.2.9
 - dask
 
@@ -54,6 +53,7 @@ dependencies:
 - descartes
 - rasterio!=1.2.10
 
+
 - pip:
   - tsam>=1.1.0
-  - git+https://github.com/PyPSA/PyPSA.git@master
+  - pypsa>=0.25.1

matplotlibrc

@@ -4,3 +4,4 @@
 font.family: sans-serif
 font.sans-serif: Ubuntu, DejaVu Sans
 image.cmap: viridis
+figure.autolayout : True

rules/build_electricity.smk

@@ -62,6 +62,9 @@ rule base_network:
     params:
         countries=config["countries"],
         snapshots=config["snapshots"],
+        lines=config["lines"],
+        links=config["links"],
+        transformers=config["transformers"],
     input:
         eg_buses="data/entsoegridkit/buses.csv",
         eg_lines="data/entsoegridkit/lines.csv",
@@ -203,75 +206,117 @@ rule build_ship_raster:
         "../scripts/build_ship_raster.py"
 
 
-# rule build_renewable_profiles:
-#     params:
-#         renewable=config["renewable"],
-#     input:
-#         base_network=RESOURCES + "networks/base.nc",
-#         corine=ancient("data/bundle/corine/g250_clc06_V18_5.tif"),
-#         natura=lambda w: (
-#             RESOURCES + "natura.tiff"
-#             if config["renewable"][w.technology]["natura"]
-#             else []
-#         ),
-#         gebco=ancient(
-#             lambda w: (
-#                 "data/bundle/GEBCO_2014_2D.nc"
-#                 if config["renewable"][w.technology].get("max_depth")
-#                 else []
-#             )
-#         ),
-#         ship_density=lambda w: (
-#             RESOURCES + "shipdensity_raster.tif"
-#             if "ship_threshold" in config["renewable"][w.technology].keys()
-#             else []
-#         ),
-#         country_shapes=RESOURCES + "country_shapes.geojson",
-#         offshore_shapes=RESOURCES + "offshore_shapes.geojson",
-#         regions=lambda w: (
-#             RESOURCES + "regions_onshore.geojson"
-#             if w.technology in ("onwind", "solar")
-#             else RESOURCES + "regions_offshore.geojson"
-#         ),
-#         cutout=lambda w: "cutouts/"
-#         + CDIR
-#         + config["renewable"][w.technology]["cutout"]
-#         + ".nc",
-#     output:
-#         profile=RESOURCES + "profile_{technology}.nc",
-#     log:
-#         LOGS + "build_renewable_profile_{technology}.log",
-#     benchmark:
-#         BENCHMARKS + "build_renewable_profiles_{technology}"
-#     threads: ATLITE_NPROCESSES
-#     resources:
-#         mem_mb=ATLITE_NPROCESSES * 5000,
-#     wildcard_constraints:
-#         technology="(?!hydro).*",  # Any technology other than hydro
-#     conda:
-#         "../envs/environment.yaml"
-#     script:
-#         "../scripts/build_renewable_profiles.py"
+rule build_renewable_profiles:
+    params:
+        renewable=config["renewable"],
+    input:
+        base_network=RESOURCES + "networks/base.nc",
+        corine=ancient("data/bundle/corine/g250_clc06_V18_5.tif"),
+        natura=lambda w: (
+            RESOURCES + "natura.tiff"
+            if config["renewable"][w.technology]["natura"]
+            else []
+        ),
+        gebco=ancient(
+            lambda w: (
+                "data/bundle/GEBCO_2014_2D.nc"
+                if config["renewable"][w.technology].get("max_depth")
+                else []
+            )
+        ),
+        ship_density=lambda w: (
+            RESOURCES + "shipdensity_raster.tif"
+            if "ship_threshold" in config["renewable"][w.technology].keys()
+            else []
+        ),
+        country_shapes=RESOURCES + "country_shapes.geojson",
+        offshore_shapes=RESOURCES + "offshore_shapes.geojson",
+        regions=lambda w: (
+            RESOURCES + "regions_onshore.geojson"
+            if w.technology in ("onwind", "solar")
+            else RESOURCES + "regions_offshore.geojson"
+        ),
+        cutout=lambda w: "cutouts/"
+        + CDIR
+        + config["renewable"][w.technology]["cutout"]
+        + ".nc",
+    output:
+        profile=RESOURCES + "profile_{technology}.nc",
+    log:
+        LOGS + "build_renewable_profile_{technology}.log",
+    benchmark:
+        BENCHMARKS + "build_renewable_profiles_{technology}"
+    threads: ATLITE_NPROCESSES
+    resources:
+        mem_mb=ATLITE_NPROCESSES * 5000,
+    wildcard_constraints:
+        technology="(?!hydro).*",  # Any technology other than hydro
+    conda:
+        "../envs/environment.yaml"
+    script:
+        "../scripts/build_renewable_profiles.py"
 
 
-# rule build_hydro_profile:
-#     params:
-#         hydro=config["renewable"]["hydro"],
-#         countries=config["countries"],
-#     input:
-#         country_shapes=RESOURCES + "country_shapes.geojson",
-#         eia_hydro_generation="data/eia_hydro_annual_generation.csv",
-#         cutout=f"cutouts/" + CDIR + config["renewable"]["hydro"]["cutout"] + ".nc",
-#     output:
-#         RESOURCES + "profile_hydro.nc",
-#     log:
-#         LOGS + "build_hydro_profile.log",
-#     resources:
-#         mem_mb=5000,
-#     conda:
-#         "../envs/environment.yaml"
-#     script:
-#         "../scripts/build_hydro_profile.py"
+rule build_monthly_prices:
+    input:
+        co2_price_raw="data/validation/emission-spot-primary-market-auction-report-2019-data.xls",
+        fuel_price_raw="data/validation/energy-price-trends-xlsx-5619002.xlsx",
+    output:
+        co2_price=RESOURCES + "co2_price.csv",
+        fuel_price=RESOURCES + "monthly_fuel_price.csv",
+    log:
+        LOGS + "build_monthly_prices.log",
+    threads: 1
+    resources:
+        mem_mb=5000,
+    conda:
+        "../envs/environment.yaml"
+    script:
+        "../scripts/build_monthly_prices.py"
+
+
+rule build_hydro_profile:
+    params:
+        hydro=config["renewable"]["hydro"],
+        countries=config["countries"],
+    input:
+        country_shapes=RESOURCES + "country_shapes.geojson",
+        eia_hydro_generation="data/eia_hydro_annual_generation.csv",
+        cutout=f"cutouts/" + CDIR + config["renewable"]["hydro"]["cutout"] + ".nc",
+    output:
+        RESOURCES + "profile_hydro.nc",
+    log:
+        LOGS + "build_hydro_profile.log",
+    resources:
+        mem_mb=5000,
+    conda:
+        "../envs/environment.yaml"
+    script:
+        "../scripts/build_hydro_profile.py"
+
+
+if config["lines"]["dynamic_line_rating"]["activate"]:
+
+    rule build_line_rating:
+        input:
+            base_network=RESOURCES + "networks/base.nc",
+            cutout="cutouts/"
+            + CDIR
+            + config["lines"]["dynamic_line_rating"]["cutout"]
+            + ".nc",
+        output:
+            output=RESOURCES + "networks/line_rating.nc",
+        log:
+            LOGS + "build_line_rating.log",
+        benchmark:
+            BENCHMARKS + "build_line_rating"
+        threads: ATLITE_NPROCESSES
+        resources:
+            mem_mb=ATLITE_NPROCESSES * 1000,
+        conda:
+            "../envs/environment.yaml"
+        script:
+            "../scripts/build_line_rating.py"
 
 
 rule add_electricity:
@@ -281,7 +326,7 @@ rule add_electricity:
         countries=config["countries"],
         renewable=config["renewable"],
         electricity=config["electricity"],
-        conventional=config.get("conventional", {}),
+        conventional=config["conventional"],
         costs=config["costs"],
     input:
         **{
@@ -291,15 +336,21 @@ rule add_electricity:
         **{
             f"conventional_{carrier}_{attr}": fn
             for carrier, d in config.get("conventional", {None: {}}).items()
+            if carrier in config["electricity"]["conventional_carriers"]
             for attr, fn in d.items()
             if str(fn).startswith("data/")
         },
         base_network=RESOURCES + "networks/base.nc",
+        line_rating=RESOURCES + "networks/line_rating.nc"
+        if config["lines"]["dynamic_line_rating"]["activate"]
+        else RESOURCES + "networks/base.nc",
         tech_costs=COSTS,
         regions=RESOURCES + "regions_onshore.geojson",
         powerplants=RESOURCES + "powerplants.csv",
         hydro_capacities=ancient("data/bundle/hydro_capacities.csv"),
         geth_hydro_capacities="data/geth2015_hydro_capacities.csv",
+        unit_commitment="data/unit_commitment.csv",
+        fuel_price=RESOURCES + "monthly_fuel_price.csv",
         load=RESOURCES + "load.csv",
         nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
     output:
@@ -310,7 +361,7 @@ rule add_electricity:
         BENCHMARKS + "add_electricity"
     threads: 1
     resources:
-        mem_mb=5000,
+        mem_mb=10000,
     conda:
         "../envs/environment.yaml"
     script:
@@ -344,7 +395,7 @@ rule simplify_network:
         BENCHMARKS + "simplify_network/elec_s{simpl}"
     threads: 1
     resources:
-        mem_mb=4000,
+        mem_mb=12000,
     conda:
         "../envs/environment.yaml"
     script:
@@ -385,7 +436,7 @@ rule cluster_network:
         BENCHMARKS + "cluster_network/elec_s{simpl}_{clusters}"
     threads: 1
     resources:
-        mem_mb=6000,
+        mem_mb=10000,
     conda:
         "../envs/environment.yaml"
     script:
@@ -408,7 +459,7 @@ rule add_extra_components:
         BENCHMARKS + "add_extra_components/elec_s{simpl}_{clusters}_ec"
     threads: 1
     resources:
-        mem_mb=3000,
+        mem_mb=4000,
     conda:
         "../envs/environment.yaml"
     script:
@@ -427,6 +478,7 @@ rule prepare_network:
     input:
         RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc",
         tech_costs=COSTS,
+        co2_price=RESOURCES + "co2_price.csv",
     output:
         RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
     log:

rules/build_sector.smk

@@ -86,7 +86,7 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
     rule build_gas_input_locations:
         input:
             lng=HTTP.remote(
-                "https://globalenergymonitor.org/wp-content/uploads/2022/09/Europe-Gas-Tracker-August-2022.xlsx",
+                "https://globalenergymonitor.org/wp-content/uploads/2023/07/Europe-Gas-Tracker-2023-03-v3.xlsx",
                 keep_local=True,
             ),
             entry="data/gas_network/scigrid-gas/data/IGGIELGN_BorderPoints.geojson",
@@ -295,7 +295,7 @@ rule build_biomass_potentials:
         "../scripts/build_biomass_potentials.py"
 
 
-if config["sector"]["biomass_transport"]:
+if config["sector"]["biomass_transport"] or config["sector"]["biomass_spatial"]:
 
     rule build_biomass_transport_costs:
         input:
@@ -320,9 +320,8 @@ if config["sector"]["biomass_transport"]:
     build_biomass_transport_costs_output = rules.build_biomass_transport_costs.output
 
 
-if not config["sector"]["biomass_transport"]:
+if not (config["sector"]["biomass_transport"] or config["sector"]["biomass_spatial"]):
     # this is effecively an `else` statement which is however not liked by snakefmt
-
     build_biomass_transport_costs_output = {}
 
 
@@ -724,7 +723,6 @@ rule prepare_sector_network:
         **build_biomass_transport_costs_output,
         **gas_infrastructure,
         **build_sequestration_potentials_output,
-        overrides="data/override_component_attrs",
         network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
         energy_totals_name=RESOURCES + "energy_totals.csv",
         eurostat=input_eurostat,

rules/collect.smk

@@ -82,3 +82,18 @@ rule plot_networks:
             + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf",
             **config["scenario"]
         ),
+
+
+rule validate_elec_networks:
+    input:
+        expand(
+            RESULTS
+            + "figures/.statistics_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}",
+            **config["scenario"]
+        ),
+        expand(
+            RESULTS
+            + "figures/.validation_{kind}_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}",
+            **config["scenario"],
+            kind=["production", "prices", "cross_border"]
+        ),

rules/common.smk

@@ -15,8 +15,8 @@ def memory(w):
         if m is not None:
             factor *= int(m.group(1)) / 8760
             break
-    if w.clusters.endswith("m"):
-        return int(factor * (18000 + 180 * int(w.clusters[:-1])))
+    if w.clusters.endswith("m") or w.clusters.endswith("c"):
+        return int(factor * (55000 + 600 * int(w.clusters[:-1])))
     elif w.clusters == "all":
         return int(factor * (18000 + 180 * 4000))
     else:

rules/postprocess.smk

@@ -13,7 +13,6 @@ rule plot_network:
         foresight=config["foresight"],
         plotting=config["plotting"],
     input:
-        overrides="data/override_component_attrs",
         network=RESULTS
         + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
         regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
@@ -35,12 +34,38 @@ rule plot_network:
     script:
         "../scripts/plot_network.py"
 
+rule plot_network_perfect:
+    params:
+        foresight=config["foresight"],
+        plotting=config["plotting"],
+    input:
+        network=RESULTS
+        + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
+        regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
+    output:
+        map=RESULTS
+        + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{year}.pdf",
+        today=RESULTS
+        + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{year}-today.pdf",
+    threads: 2
+    resources:
+        mem_mb=10000,
+    benchmark:
+        BENCHMARKS
+        + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{year}_brownfield_all_years_benchmark.txt",
+    conda:
+        "../envs/environment.yaml"
+    script:
+        "../scripts/plot_network.py"
+
+
+
 
 rule copy_config:
     params:
         RDIR=RDIR,
     output:
-        RESULTS + "config/config.yaml",
+        RESULTS + "config.yaml",
     threads: 1
     resources:
         mem_mb=1000,
@@ -52,22 +77,6 @@ rule copy_config:
         "../scripts/copy_config.py"
 
 
-rule copy_conda_env:
-    output:
-        RESULTS + "config/environment.yaml",
-    threads: 1
-    resources:
-        mem_mb=500,
-    log:
-        LOGS + "copy_conda_env.log",
-    benchmark:
-        BENCHMARKS + "copy_conda_env"
-    conda:
-        "../envs/environment.yaml"
-    shell:
-        "conda env export -f {output} --no-builds"
-
-
 rule make_summary:
     params:
         foresight=config["foresight"],
@@ -76,7 +85,6 @@ rule make_summary:
         scenario=config["scenario"],
         RDIR=RDIR,
     input:
-        overrides="data/override_component_attrs",
         networks=expand(
             RESULTS
             + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
@@ -147,3 +155,34 @@ rule plot_summary:
         "../envs/environment.yaml"
     script:
         "../scripts/plot_summary.py"
+
+
+STATISTICS_BARPLOTS = [
+    "capacity_factor",
+    "installed_capacity",
+    "optimal_capacity",
+    "capital_expenditure",
+    "operational_expenditure",
+    "curtailment",
+    "supply",
+    "withdrawal",
+    "market_value",
+]
+
+
+rule plot_elec_statistics:
+    params:
+        plotting=config["plotting"],
+        barplots=STATISTICS_BARPLOTS,
+    input:
+        network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
+    output:
+        **{
+            f"{plot}_bar": RESULTS
+            + f"figures/statistics_{plot}_bar_elec_s{{simpl}}_{{clusters}}_ec_l{{ll}}_{{opts}}.pdf"
+            for plot in STATISTICS_BARPLOTS
+        },
+        barplots_touch=RESULTS
+        + "figures/.statistics_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}",
+    script:
+        "../scripts/plot_statistics.py"

rules/retrieve.smk

@@ -158,7 +158,11 @@ if config["enable"]["retrieve"]:
     rule retrieve_electricity_demand:
         input:
             HTTP.remote(
-                "data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv",
+                "data.open-power-system-data.org/time_series/{version}/time_series_60min_singleindex.csv".format(
+                version="2019-06-05"
+                    if config["snapshots"]["end"] < "2019"
+                    else "2020-10-06"
+                ),
                 keep_local=True,
                 static=True,
             ),
@@ -191,3 +195,39 @@ if config["enable"]["retrieve"]:
         retries: 2
         run:
             move(input[0], output[0])
+
+
+if config["enable"]["retrieve"]:
+
+    rule retrieve_monthly_co2_prices:
+        input:
+            HTTP.remote(
+                "https://www.eex.com/fileadmin/EEX/Downloads/EUA_Emission_Spot_Primary_Market_Auction_Report/Archive_Reports/emission-spot-primary-market-auction-report-2019-data.xls",
+                keep_local=True,
+                static=True,
+            ),
+        output:
+            "data/validation/emission-spot-primary-market-auction-report-2019-data.xls",
+        log:
+            LOGS + "retrieve_monthly_co2_prices.log",
+        resources:
+            mem_mb=5000,
+        retries: 2
+        run:
+            move(input[0], output[0])
+
+
+if config["enable"]["retrieve"]:
+
+    rule retrieve_monthly_fuel_prices:
+        output:
+            "data/validation/energy-price-trends-xlsx-5619002.xlsx",
+        log:
+            LOGS + "retrieve_monthly_fuel_prices.log",
+        resources:
+            mem_mb=5000,
+        retries: 2
+        conda:
+            "../envs/environment.yaml"
+        script:
+            "../scripts/retrieve_monthly_fuel_prices.py"

rules/solve_electricity.smk

@@ -13,6 +13,7 @@ rule solve_network:
         ),
     input:
         network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
+        config=RESULTS + "config.yaml",
     output:
         network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
     log:
@@ -26,6 +27,7 @@ rule solve_network:
     threads: 4
     resources:
         mem_mb=memory,
+        walltime=config["solving"].get("walltime", "12:00:00"),
     shadow:
         "minimal"
     conda:
@@ -55,7 +57,8 @@ rule solve_operations_network:
         )
     threads: 4
     resources:
-        mem_mb=(lambda w: 5000 + 372 * int(w.clusters)),
+        mem_mb=(lambda w: 10000 + 372 * int(w.clusters)),
+        walltime=config["solving"].get("walltime", "12:00:00"),
     shadow:
         "minimal"
     conda:

rules/solve_myopic.smk

@@ -10,7 +10,6 @@ rule add_existing_baseyear:
         existing_capacities=config["existing_capacities"],
         costs=config["costs"],
     input:
-        overrides="data/override_component_attrs",
         network=RESULTS
         + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
         powerplants=RESOURCES + "powerplants.csv",
@@ -52,7 +51,6 @@ rule add_brownfield:
         H2_retrofit_capacity_per_CH4=config["sector"]["H2_retrofit_capacity_per_CH4"],
         threshold_capacity=config["existing_capacities"]["threshold_capacity"],
     input:
-        overrides="data/override_component_attrs",
         network=RESULTS
         + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
         network_p=solved_previous_horizon,  #solved network at previous time step
@@ -91,11 +89,10 @@ rule solve_sector_network_myopic:
             "co2_sequestration_potential", 200
         ),
     input:
-        overrides="data/override_component_attrs",
         network=RESULTS
         + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
         costs="data/costs_{planning_horizons}.csv",
-        config=RESULTS + "config/config.yaml",
+        config=RESULTS + "config.yaml",
     output:
         RESULTS
         + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
@@ -109,6 +106,7 @@ rule solve_sector_network_myopic:
     threads: 4
     resources:
         mem_mb=config["solving"]["mem"],
+        walltime=config["solving"].get("walltime", "12:00:00"),
     benchmark:
         (
             BENCHMARKS

rules/solve_overnight.smk

@@ -12,12 +12,9 @@ rule solve_sector_network:
             "co2_sequestration_potential", 200
         ),
     input:
-        overrides="data/override_component_attrs",
         network=RESULTS
         + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-        costs="data/costs_{}.csv".format(config["costs"]["year"]),
-        config=RESULTS + "config/config.yaml",
-        #env=RDIR + 'config/environment.yaml',
+        config=RESULTS + "config.yaml",
     output:
         RESULTS
         + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
@@ -31,6 +28,7 @@ rule solve_sector_network:
     threads: config["solving"]["solver"].get("threads", 4)
     resources:
         mem_mb=config["solving"]["mem"],
+        walltime=config["solving"].get("walltime", "12:00:00"),
     benchmark:
         (
             RESULTS

rules/solve_perfect.smk

@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: MIT
 rule add_existing_baseyear:
     input:
-        overrides="data/override_component_attrs",
         network=RESULTS
         + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
         powerplants=RESOURCES + "powerplants.csv",
@@ -41,7 +40,6 @@ rule add_existing_baseyear:
 
 rule add_brownfield:
     input:
-        overrides="data/override_component_attrs",
         network=RESULTS
         + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
         network_p=solved_previous_horizon,  #solved network at previous time step
@@ -82,7 +80,6 @@ rule prepare_perfect_foresight:
             + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_"
             + "{}.nc".format(str(config["scenario"]["planning_horizons"][0]))
         ),
-        overrides="data/override_component_attrs",
     output:
         RESULTS
         + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
@@ -105,7 +102,6 @@ rule prepare_perfect_foresight:
 
 rule solve_sector_network_perfect:
     input:
-        overrides="data/override_component_attrs",
         network=RESULTS
         + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
         costs="data/costs_2030.csv",
@@ -148,7 +144,6 @@ rule make_summary_perfect:
             for ll in config["scenario"]["ll"]
         },
         costs="data/costs_2020.csv",
-        overrides="data/override_component_attrs",
     output:
         nodal_costs=RESULTS + "csvs/nodal_costs.csv",
         nodal_capacities=RESULTS + "csvs/nodal_capacities.csv",

rules/validate.smk

@@ -0,0 +1,117 @@
+# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+PRODUCTION_PLOTS = [
+    "production_bar",
+    "production_deviation_bar",
+    "seasonal_operation_area",
+]
+CROSS_BORDER_PLOTS = ["trade_time_series", "cross_border_bar"]
+PRICES_PLOTS = ["price_bar", "price_line"]
+
+
+rule build_electricity_production:
+    """
+    This rule builds the electricity production for each country and technology from ENTSO-E data.
+    The data is used for validation of the optimization results.
+    """
+    params:
+        snapshots=config["snapshots"],
+        countries=config["countries"],
+    output:
+        RESOURCES + "historical_electricity_production.csv",
+    log:
+        LOGS + "build_electricity_production.log",
+    resources:
+        mem_mb=5000,
+    script:
+        "../scripts/build_electricity_production.py"
+
+
+rule build_cross_border_flows:
+    """
+    This rule builds the cross-border flows from ENTSO-E data.
+    The data is used for validation of the optimization results.
+    """
+    params:
+        snapshots=config["snapshots"],
+        countries=config["countries"],
+    input:
+        network=RESOURCES + "networks/base.nc",
+    output:
+        RESOURCES + "historical_cross_border_flows.csv",
+    log:
+        LOGS + "build_cross_border_flows.log",
+    resources:
+        mem_mb=5000,
+    script:
+        "../scripts/build_cross_border_flows.py"
+
+
+rule build_electricity_prices:
+    """
+    This rule builds the electricity prices from ENTSO-E data.
+    The data is used for validation of the optimization results.
+    """
+    params:
+        snapshots=config["snapshots"],
+        countries=config["countries"],
+    output:
+        RESOURCES + "historical_electricity_prices.csv",
+    log:
+        LOGS + "build_electricity_prices.log",
+    resources:
+        mem_mb=5000,
+    script:
+        "../scripts/build_electricity_prices.py"
+
+
+rule plot_validation_electricity_production:
+    input:
+        network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
+        electricity_production=RESOURCES + "historical_electricity_production.csv",
+    output:
+        **{
+            plot: RESULTS
+            + f"figures/validation_{plot}_elec_s{{simpl}}_{{clusters}}_ec_l{{ll}}_{{opts}}.pdf"
+            for plot in PRODUCTION_PLOTS
+        },
+        plots_touch=RESULTS
+        + "figures/.validation_production_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}",
+    script:
+        "../scripts/plot_validation_electricity_production.py"
+
+
+rule plot_validation_cross_border_flows:
+    params:
+        countries=config["countries"],
+    input:
+        network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
+        cross_border_flows=RESOURCES + "historical_cross_border_flows.csv",
+    output:
+        **{
+            plot: RESULTS
+            + f"figures/validation_{plot}_elec_s{{simpl}}_{{clusters}}_ec_l{{ll}}_{{opts}}.pdf"
+            for plot in CROSS_BORDER_PLOTS
+        },
+        plots_touch=RESULTS
+        + "figures/.validation_cross_border_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}",
+    script:
+        "../scripts/plot_validation_cross_border_flows.py"
+
+
+rule plot_validation_electricity_prices:
+    input:
+        network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
+        electricity_prices=RESOURCES + "historical_electricity_prices.csv",
+    output:
+        **{
+            plot: RESULTS
+            + f"figures/validation_{plot}_elec_s{{simpl}}_{{clusters}}_ec_l{{ll}}_{{opts}}.pdf"
+            for plot in PRICES_PLOTS
+        },
+        plots_touch=RESULTS
+        + "figures/.validation_prices_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}",
+    script:
+        "../scripts/plot_validation_electricity_prices.py"

scripts/_helpers.py

@@ -72,92 +72,6 @@ def configure_logging(snakemake, skip_handlers=False):
     logging.basicConfig(**kwargs)
 
 
-def load_network(import_name=None, custom_components=None):
-    """
-    Helper for importing a pypsa.Network with additional custom components.
-
-    Parameters
-    ----------
-    import_name : str
-        As in pypsa.Network(import_name)
-    custom_components : dict
-        Dictionary listing custom components.
-        For using ``snakemake.params['override_components']``
-        in ``config/config.yaml`` define:
-
-        .. code:: yaml
-
-            override_components:
-                ShadowPrice:
-                    component: ["shadow_prices","Shadow price for a global constraint.",np.nan]
-                    attributes:
-                    name: ["string","n/a","n/a","Unique name","Input (required)"]
-                    value: ["float","n/a",0.,"shadow value","Output"]
-
-    Returns
-    -------
-    pypsa.Network
-    """
-    import pypsa
-    from pypsa.descriptors import Dict
-
-    override_components = None
-    override_component_attrs = None
-
-    if custom_components is not None:
-        override_components = pypsa.components.components.copy()
-        override_component_attrs = Dict(
-            {k: v.copy() for k, v in pypsa.components.component_attrs.items()}
-        )
-        for k, v in custom_components.items():
-            override_components.loc[k] = v["component"]
-            override_component_attrs[k] = pd.DataFrame(
-                columns=["type", "unit", "default", "description", "status"]
-            )
-            for attr, val in v["attributes"].items():
-                override_component_attrs[k].loc[attr] = val
-
-    return pypsa.Network(
-        import_name=import_name,
-        override_components=override_components,
-        override_component_attrs=override_component_attrs,
-    )
-
-
-def load_network_for_plots(fn, tech_costs, config, combine_hydro_ps=True):
-    import pypsa
-    from add_electricity import load_costs, update_transmission_costs
-
-    n = pypsa.Network(fn)
-
-    n.loads["carrier"] = n.loads.bus.map(n.buses.carrier) + " load"
-    n.stores["carrier"] = n.stores.bus.map(n.buses.carrier)
-
-    n.links["carrier"] = (
-        n.links.bus0.map(n.buses.carrier) + "-" + n.links.bus1.map(n.buses.carrier)
-    )
-    n.lines["carrier"] = "AC line"
-    n.transformers["carrier"] = "AC transformer"
-
-    n.lines["s_nom"] = n.lines["s_nom_min"]
-    n.links["p_nom"] = n.links["p_nom_min"]
-
-    if combine_hydro_ps:
-        n.storage_units.loc[
-            n.storage_units.carrier.isin({"PHS", "hydro"}), "carrier"
-        ] = "hydro+PHS"
-
-    # if the carrier was not set on the heat storage units
-    # bus_carrier = n.storage_units.bus.map(n.buses.carrier)
-    # n.storage_units.loc[bus_carrier == "heat","carrier"] = "water tanks"
-
-    Nyears = n.snapshot_weightings.objective.sum() / 8760.0
-    costs = load_costs(tech_costs, config["costs"], config["electricity"], Nyears)
-    update_transmission_costs(n, costs)
-
-    return n
-
-
 def update_p_nom_max(n):
     # if extendable carriers (solar/onwind/...) have capacity >= 0,
     # e.g. existing assets from the OPSD project are included to the network,
@@ -367,34 +281,6 @@ def mock_snakemake(rulename, configfiles=[], **wildcards):
     return snakemake
 
 
-def override_component_attrs(directory):
-    """
-    Tell PyPSA that links can have multiple outputs by overriding the
-    component_attrs. This can be done for as many buses as you need with format
-    busi for i = 2,3,4,5,.... See https://pypsa.org/doc/components.html#link-
-    with-multiple-outputs-or-inputs.
-
-    Parameters
-    ----------
-    directory : string
-        Folder where component attributes to override are stored
-        analogous to ``pypsa/component_attrs``, e.g. `links.csv`.
-
-    Returns
-    -------
-    Dictionary of overridden component attributes.
-    """
-    attrs = Dict({k: v.copy() for k, v in component_attrs.items()})
-
-    for component, list_name in components.list_name.items():
-        fn = f"{directory}/{list_name}.csv"
-        if os.path.isfile(fn):
-            overrides = pd.read_csv(fn, index_col=0, na_values="n/a")
-            attrs[component] = overrides.combine_first(attrs[component])
-
-    return attrs
-
-
 def generate_periodic_profiles(dt_index, nodes, weekly_profile, localize=None):
     """
     Give a 24*7 long list of weekly hourly profiles, generate this for each

scripts/add_brownfield.py

@@ -16,7 +16,7 @@ idx = pd.IndexSlice
 
 import numpy as np
 import pypsa
-from _helpers import override_component_attrs, update_config_with_sector_opts
+from _helpers import update_config_with_sector_opts
 from add_existing_baseyear import add_build_year_to_new_assets
 
 
@@ -147,12 +147,11 @@ if __name__ == "__main__":
 
     year = int(snakemake.wildcards.planning_horizons)
 
-    overrides = override_component_attrs(snakemake.input.overrides)
-    n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
+    n = pypsa.Network(snakemake.input.network)
 
     add_build_year_to_new_assets(n, year)
 
-    n_p = pypsa.Network(snakemake.input.network_p, override_component_attrs=overrides)
+    n_p = pypsa.Network(snakemake.input.network_p)
 
     add_brownfield(n, n_p, year)
 

scripts/add_electricity.py

@@ -2,8 +2,6 @@
 # SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
-# coding: utf-8
 """
 Adds electrical generators and existing hydro storage units to a base network.
 
@@ -206,7 +204,6 @@ def load_costs(tech_costs, config, max_hours, Nyears=1.0):
         * costs["investment"]
         * Nyears
     )
-
     costs.at["OCGT", "fuel"] = costs.at["gas", "fuel"]
     costs.at["CCGT", "fuel"] = costs.at["gas", "fuel"]
 
@@ -362,7 +359,6 @@ def attach_wind_and_solar(
     n, costs, input_profiles, carriers, extendable_carriers, line_length_factor=1
 ):
     add_missing_carriers(n, carriers)
-
     for car in carriers:
         if car == "hydro":
             continue
@@ -421,26 +417,54 @@ def attach_conventional_generators(
     extendable_carriers,
     conventional_params,
     conventional_inputs,
+    unit_commitment=None,
+    fuel_price=None,
 ):
     carriers = list(set(conventional_carriers) | set(extendable_carriers["Generator"]))
     add_missing_carriers(n, carriers)
     add_co2_emissions(n, costs, carriers)
 
+    # Replace carrier "natural gas" with the respective technology (OCGT or
+    # CCGT) to align with PyPSA names of "carriers" and avoid filtering "natural
+    # gas" powerplants in ppl.query("carrier in @carriers")
+    ppl.loc[ppl["carrier"] == "natural gas", "carrier"] = ppl.loc[
+        ppl["carrier"] == "natural gas", "technology"
+    ]
+
     ppl = (
         ppl.query("carrier in @carriers")
         .join(costs, on="carrier", rsuffix="_r")
         .rename(index=lambda s: "C" + str(s))
     )
     ppl["efficiency"] = ppl.efficiency.fillna(ppl.efficiency_r)
-    ppl["marginal_cost"] = (
-        ppl.carrier.map(costs.VOM) + ppl.carrier.map(costs.fuel) / ppl.efficiency
-    )
 
-    logger.info(
-        "Adding {} generators with capacities [GW] \n{}".format(
-            len(ppl), ppl.groupby("carrier").p_nom.sum().div(1e3).round(2)
+    if unit_commitment is not None:
+        committable_attrs = ppl.carrier.isin(unit_commitment).to_frame("committable")
+        for attr in unit_commitment.index:
+            default = pypsa.components.component_attrs["Generator"].default[attr]
+            committable_attrs[attr] = ppl.carrier.map(unit_commitment.loc[attr]).fillna(
+                default
+            )
+    else:
+        committable_attrs = {}
+
+    if fuel_price is not None:
+        fuel_price = fuel_price.assign(
+            OCGT=fuel_price["gas"], CCGT=fuel_price["gas"]
+        ).drop("gas", axis=1)
+        missing_carriers = list(set(carriers) - set(fuel_price))
+        fuel_price = fuel_price.assign(**costs.fuel[missing_carriers])
+        fuel_price = fuel_price.reindex(ppl.carrier, axis=1)
+        fuel_price.columns = ppl.index
+        marginal_cost = fuel_price.div(ppl.efficiency).add(ppl.carrier.map(costs.VOM))
+    else:
+        marginal_cost = (
+            ppl.carrier.map(costs.VOM) + ppl.carrier.map(costs.fuel) / ppl.efficiency
         )
-    )
+
+    # Define generators using modified ppl DataFrame
+    caps = ppl.groupby("carrier").p_nom.sum().div(1e3).round(2)
+    logger.info(f"Adding {len(ppl)} generators with capacities [GW] \n{caps}")
 
     n.madd(
         "Generator",
@@ -451,13 +475,14 @@ def attach_conventional_generators(
         p_nom=ppl.p_nom.where(ppl.carrier.isin(conventional_carriers), 0),
         p_nom_extendable=ppl.carrier.isin(extendable_carriers["Generator"]),
         efficiency=ppl.efficiency,
-        marginal_cost=ppl.marginal_cost,
+        marginal_cost=marginal_cost,
         capital_cost=ppl.capital_cost,
         build_year=ppl.datein.fillna(0).astype(int),
         lifetime=(ppl.dateout - ppl.datein).fillna(np.inf),
+        **committable_attrs,
     )
 
-    for carrier in conventional_params:
+    for carrier in set(conventional_params) & set(carriers):
         # Generators with technology affected
         idx = n.generators.query("carrier == @carrier").index
 
@@ -591,6 +616,14 @@ def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **par
             hydro.max_hours > 0, hydro.country.map(max_hours_country)
         ).fillna(6)
 
+        flatten_dispatch = params.get("flatten_dispatch", False)
+        if flatten_dispatch:
+            buffer = params.get("flatten_dispatch_buffer", 0.2)
+            average_capacity_factor = inflow_t[hydro.index].mean() / hydro["p_nom"]
+            p_max_pu = (average_capacity_factor + buffer).clip(upper=1)
+        else:
+            p_max_pu = 1
+
         n.madd(
             "StorageUnit",
             hydro.index,
@@ -600,7 +633,7 @@ def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **par
             max_hours=hydro_max_hours,
             capital_cost=costs.at["hydro", "capital_cost"],
             marginal_cost=costs.at["hydro", "marginal_cost"],
-            p_max_pu=1.0,  # dispatch
+            p_max_pu=p_max_pu,  # dispatch
             p_min_pu=0.0,  # store
             efficiency_dispatch=costs.at["hydro", "efficiency"],
             efficiency_store=0.0,
@@ -690,13 +723,14 @@ def attach_OPSD_renewables(n, tech_map):
         {"Solar": "PV"}
     )
     df = df.query("Fueltype in @tech_map").powerplant.convert_country_to_alpha2()
+    df = df.dropna(subset=["lat", "lon"])
 
     for fueltype, carriers in tech_map.items():
         gens = n.generators[lambda df: df.carrier.isin(carriers)]
         buses = n.buses.loc[gens.bus.unique()]
         gens_per_bus = gens.groupby("bus").p_nom.count()
 
-        caps = map_country_bus(df.query("Fueltype == @fueltype and lat == lat"), buses)
+        caps = map_country_bus(df.query("Fueltype == @fueltype"), buses)
         caps = caps.groupby(["bus"]).Capacity.sum()
         caps = caps / gens_per_bus.reindex(caps.index, fill_value=1)
 
@@ -745,6 +779,30 @@ def estimate_renewable_capacities(n, year, tech_map, expansion_limit, countries)
             )
 
 
+def attach_line_rating(
+    n, rating, s_max_pu, correction_factor, max_voltage_difference, max_line_rating
+):
+    # TODO: Only considers overhead lines
+    n.lines_t.s_max_pu = (rating / n.lines.s_nom[rating.columns]) * correction_factor
+    if max_voltage_difference:
+        x_pu = (
+            n.lines.type.map(n.line_types["x_per_length"])
+            * n.lines.length
+            / (n.lines.v_nom**2)
+        )
+        # need to clip here as cap values might be below 1
+        # -> would mean the line cannot be operated at actual given pessimistic ampacity
+        s_max_pu_cap = (
+            np.deg2rad(max_voltage_difference) / (x_pu * n.lines.s_nom)
+        ).clip(lower=1)
+        n.lines_t.s_max_pu = n.lines_t.s_max_pu.clip(
+            lower=1, upper=s_max_pu_cap, axis=1
+        )
+    if max_line_rating:
+        n.lines_t.s_max_pu = n.lines_t.s_max_pu.clip(upper=max_line_rating)
+    n.lines_t.s_max_pu *= s_max_pu
+
+
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
@@ -782,6 +840,20 @@ if __name__ == "__main__":
     conventional_inputs = {
         k: v for k, v in snakemake.input.items() if k.startswith("conventional_")
     }
+
+    if params.conventional["unit_commitment"]:
+        unit_commitment = pd.read_csv(snakemake.input.unit_commitment, index_col=0)
+    else:
+        unit_commitment = None
+
+    if params.conventional["dynamic_fuel_price"]:
+        fuel_price = pd.read_csv(
+            snakemake.input.fuel_price, index_col=0, header=0, parse_dates=True
+        )
+        fuel_price = fuel_price.reindex(n.snapshots).fillna(method="ffill")
+    else:
+        fuel_price = None
+
     attach_conventional_generators(
         n,
         costs,
@@ -790,6 +862,8 @@ if __name__ == "__main__":
         extendable_carriers,
         params.conventional,
         conventional_inputs,
+        unit_commitment=unit_commitment,
+        fuel_price=fuel_price,
     )
 
     attach_wind_and_solar(
@@ -802,15 +876,16 @@ if __name__ == "__main__":
     )
 
     if "hydro" in renewable_carriers:
-        para = params.renewable["hydro"]
+        p = params.renewable["hydro"]
+        carriers = p.pop("carriers", [])
         attach_hydro(
             n,
             costs,
             ppl,
             snakemake.input.profile_hydro,
             snakemake.input.hydro_capacities,
-            para.pop("carriers", []),
-            **para,
+            carriers,
+            **p,
         )
 
     estimate_renewable_caps = params.electricity["estimate_renewable_capacities"]
@@ -827,6 +902,23 @@ if __name__ == "__main__":
 
     update_p_nom_max(n)
 
+    line_rating_config = snakemake.config["lines"]["dynamic_line_rating"]
+    if line_rating_config["activate"]:
+        rating = xr.open_dataarray(snakemake.input.line_rating).to_pandas().transpose()
+        s_max_pu = snakemake.config["lines"]["s_max_pu"]
+        correction_factor = line_rating_config["correction_factor"]
+        max_voltage_difference = line_rating_config["max_voltage_difference"]
+        max_line_rating = line_rating_config["max_line_rating"]
+
+        attach_line_rating(
+            n,
+            rating,
+            s_max_pu,
+            correction_factor,
+            max_voltage_difference,
+            max_line_rating,
+        )
+
     sanitize_carriers(n, snakemake.config)
 
     n.meta = snakemake.config

scripts/add_existing_baseyear.py

@@ -21,7 +21,7 @@ import country_converter as coco
 import numpy as np
 import pypsa
 import xarray as xr
-from _helpers import override_component_attrs, update_config_with_sector_opts
+from _helpers import update_config_with_sector_opts
 from add_electricity import sanitize_carriers
 from prepare_sector_network import cluster_heat_buses, define_spatial, prepare_costs
 
@@ -129,10 +129,14 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas
         "Oil": "oil",
         "OCGT": "OCGT",
         "CCGT": "CCGT",
-        "Natural Gas": "gas",
         "Bioenergy": "urban central solid biomass CHP",
     }
 
+    # Replace Fueltype "Natural Gas" with the respective technology (OCGT or CCGT)
+    df_agg.loc[df_agg["Fueltype"] == "Natural Gas", "Fueltype"] = df_agg.loc[
+        df_agg["Fueltype"] == "Natural Gas", "Technology"
+    ]
+
     fueltype_to_drop = [
         "Hydro",
         "Wind",
@@ -601,12 +605,13 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake(
             "add_existing_baseyear",
+            configfiles="config/test/config.myopic.yaml",
             simpl="",
-            clusters="45",
-            ll="v1.0",
+            clusters="5",
+            ll="v1.5",
             opts="",
-            sector_opts="8760H-T-H-B-I-A-solar+p3-dist1",
-            planning_horizons=2020,
+            sector_opts="24H-T-H-B-I-A-solar+p3-dist1",
+            planning_horizons=2030,
         )
 
     logging.basicConfig(level=snakemake.config["logging"]["level"])
@@ -618,8 +623,8 @@ if __name__ == "__main__":
 
     baseyear = snakemake.params.baseyear
 
-    overrides = override_component_attrs(snakemake.input.overrides)
-    n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
+    n = pypsa.Network(snakemake.input.network)
+
     # define spatial resolution of carriers
     spatial = define_spatial(n.buses[n.buses.carrier == "AC"].index, options)
     add_build_year_to_new_assets(n, baseyear)

scripts/base_network.py

@@ -337,7 +337,7 @@ def _load_lines_from_eg(buses, eg_lines):
     )
 
     lines["length"] /= 1e3
-
+    lines["carrier"] = "AC"
     lines = _remove_dangling_branches(lines, buses)
 
     return lines

scripts/build_cross_border_flows.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+import logging
+
+import pandas as pd
+import pypsa
+from _helpers import configure_logging
+from entsoe import EntsoePandasClient
+from entsoe.exceptions import InvalidBusinessParameterError, NoMatchingDataError
+from requests import HTTPError
+
+logger = logging.getLogger(__name__)
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake("build_cross_border_flows")
+    configure_logging(snakemake)
+
+    api_key = snakemake.config["private"]["keys"]["entsoe_api"]
+    client = EntsoePandasClient(api_key=api_key)
+
+    n = pypsa.Network(snakemake.input.network)
+    start = pd.Timestamp(snakemake.params.snapshots["start"], tz="Europe/Brussels")
+    end = pd.Timestamp(snakemake.params.snapshots["end"], tz="Europe/Brussels")
+
+    branches = n.branches().query("carrier in ['AC', 'DC']")
+    c = n.buses.country
+    branch_countries = pd.concat([branches.bus0.map(c), branches.bus1.map(c)], axis=1)
+    branch_countries = branch_countries.query("bus0 != bus1")
+    branch_countries = branch_countries.apply(sorted, axis=1, result_type="broadcast")
+    country_pairs = branch_countries.drop_duplicates().reset_index(drop=True)
+
+    flows = []
+    unavailable_borders = []
+    for from_country, to_country in country_pairs.values:
+        try:
+            flow_directed = client.query_crossborder_flows(
+                from_country, to_country, start=start, end=end
+            )
+            flow_reverse = client.query_crossborder_flows(
+                to_country, from_country, start=start, end=end
+            )
+            flow = (flow_directed - flow_reverse).rename(
+                f"{from_country} - {to_country}"
+            )
+            flow = flow.tz_localize(None).resample("1h").mean()
+            flow = flow.loc[start.tz_localize(None) : end.tz_localize(None)]
+            flows.append(flow)
+        except (HTTPError, NoMatchingDataError, InvalidBusinessParameterError):
+            unavailable_borders.append(f"{from_country}-{to_country}")
+
+    if unavailable_borders:
+        logger.warning(
+            "Historical electricity cross-border flows for countries"
+            f" {', '.join(unavailable_borders)} not available."
+        )
+
+    flows = pd.concat(flows, axis=1)
+    flows.to_csv(snakemake.output[0])

scripts/build_electricity_demand.py

@@ -80,11 +80,9 @@ def load_timeseries(fn, years, countries, powerstatistics=True):
     def rename(s):
         return s[: -len(pattern)]
 
-    def date_parser(x):
-        return dateutil.parser.parse(x, ignoretz=True)
-
     return (
-        pd.read_csv(fn, index_col=0, parse_dates=[0], date_parser=date_parser)
+        pd.read_csv(fn, index_col=0, parse_dates=[0])
+        .tz_localize(None)
         .filter(like=pattern)
         .rename(columns=rename)
         .dropna(how="all", axis=0)
@@ -168,6 +166,7 @@ def manual_adjustment(load, fn_load, powerstatistics):
      by the corresponding ratio of total energy consumptions reported by
      IEA Data browser [0] for the year 2013.
 
+
      2. For the ENTSOE transparency load data (if powerstatistics is False)
 
      Albania (AL) and Macedonia (MK) do not exist in the data set. Both get the
@@ -176,6 +175,9 @@ def manual_adjustment(load, fn_load, powerstatistics):
 
      [0] https://www.iea.org/data-and-statistics?country=WORLD&fuel=Electricity%20and%20heat&indicator=TotElecCons
 
+    Bosnia and Herzegovina (BA) does not exist in the data set for 2019. It gets the
+    electricity consumption data from Croatia (HR) for the year 2019, scaled by the
+    factors derived from https://energy.at-site.be/eurostat-2021/
 
      Parameters
      ----------
@@ -264,9 +266,17 @@ def manual_adjustment(load, fn_load, powerstatistics):
                 load["AL"] = load.ME * (5.7 / 2.9)
             if "MK" not in load and "MK" in countries:
                 load["MK"] = load.ME * (6.7 / 2.9)
+            if "BA" not in load and "BA" in countries:
+                load["BA"] = load.HR * (11.0 / 16.2)
         copy_timeslice(
             load, "BG", "2018-10-27 21:00", "2018-10-28 22:00", Delta(weeks=1)
         )
+        copy_timeslice(
+            load, "LU", "2019-01-02 11:00", "2019-01-05 05:00", Delta(weeks=-1)
+        )
+        copy_timeslice(
+            load, "LU", "2019-02-05 20:00", "2019-02-06 19:00", Delta(weeks=-1)
+        )
 
     return load
 
@@ -291,6 +301,9 @@ if __name__ == "__main__":
     if snakemake.params.load["manual_adjustments"]:
         load = manual_adjustment(load, snakemake.input[0], powerstatistics)
 
+    if load.empty:
+        logger.warning("Build electricity demand time series is empty.")
+
     logger.info(f"Linearly interpolate gaps of size {interpolate_limit} and less.")
     load = load.interpolate(method="linear", limit=interpolate_limit)
 

scripts/build_electricity_prices.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+import logging
+
+import pandas as pd
+from _helpers import configure_logging
+from entsoe import EntsoePandasClient
+from entsoe.exceptions import NoMatchingDataError
+
+logger = logging.getLogger(__name__)
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake("build_cross_border_flows")
+    configure_logging(snakemake)
+
+    api_key = snakemake.config["private"]["keys"]["entsoe_api"]
+    client = EntsoePandasClient(api_key=api_key)
+
+    start = pd.Timestamp(snakemake.params.snapshots["start"], tz="Europe/Brussels")
+    end = pd.Timestamp(snakemake.params.snapshots["end"], tz="Europe/Brussels")
+
+    countries = snakemake.params.countries
+
+    prices = []
+    unavailable_countries = []
+
+    for country in countries:
+        country_code = country
+
+        try:
+            gen = client.query_day_ahead_prices(country, start=start, end=end)
+            gen = gen.tz_localize(None).resample("1h").mean()
+            gen = gen.loc[start.tz_localize(None) : end.tz_localize(None)]
+            prices.append(gen)
+        except NoMatchingDataError:
+            unavailable_countries.append(country)
+
+    if unavailable_countries:
+        logger.warning(
+            f"Historical electricity prices for countries {', '.join(unavailable_countries)} not available."
+        )
+
+    keys = [c for c in countries if c not in unavailable_countries]
+    prices = pd.concat(prices, keys=keys, axis=1)
+    prices.to_csv(snakemake.output[0])

scripts/build_electricity_production.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+import logging
+
+import pandas as pd
+from _helpers import configure_logging
+from entsoe import EntsoePandasClient
+from entsoe.exceptions import NoMatchingDataError
+
+logger = logging.getLogger(__name__)
+
+
+carrier_grouper = {
+    "Waste": "Biomass",
+    "Hydro Pumped Storage": "Hydro",
+    "Hydro Water Reservoir": "Hydro",
+    "Hydro Run-of-river and poundage": "Run of River",
+    "Fossil Coal-derived gas": "Gas",
+    "Fossil Gas": "Gas",
+    "Fossil Oil": "Oil",
+    "Fossil Oil shale": "Oil",
+    "Fossil Brown coal/Lignite": "Lignite",
+    "Fossil Peat": "Lignite",
+    "Fossil Hard coal": "Coal",
+    "Wind Onshore": "Onshore Wind",
+    "Wind Offshore": "Offshore Wind",
+    "Other renewable": "Other",
+    "Marine": "Other",
+}
+
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake("build_electricity_production")
+    configure_logging(snakemake)
+
+    api_key = snakemake.config["private"]["keys"]["entsoe_api"]
+    client = EntsoePandasClient(api_key=api_key)
+
+    start = pd.Timestamp(snakemake.params.snapshots["start"], tz="Europe/Brussels")
+    end = pd.Timestamp(snakemake.params.snapshots["end"], tz="Europe/Brussels")
+
+    countries = snakemake.params.countries
+
+    generation = []
+    unavailable_countries = []
+
+    for country in countries:
+        country_code = country
+
+        try:
+            gen = client.query_generation(country, start=start, end=end, nett=True)
+            gen = gen.tz_localize(None).resample("1h").mean()
+            gen = gen.loc[start.tz_localize(None) : end.tz_localize(None)]
+            gen = gen.rename(columns=carrier_grouper).groupby(level=0, axis=1).sum()
+            generation.append(gen)
+        except NoMatchingDataError:
+            unavailable_countries.append(country)
+
+    if unavailable_countries:
+        logger.warning(
+            f"Historical electricity production for countries {', '.join(unavailable_countries)} not available."
+        )
+
+    keys = [c for c in countries if c not in unavailable_countries]
+    generation = pd.concat(generation, keys=keys, axis=1)
+    generation.to_csv(snakemake.output[0])

scripts/build_industrial_distribution_key.py

@@ -93,6 +93,20 @@ def prepare_hotmaps_database(regions):
     gdf.rename(columns={"index_right": "bus"}, inplace=True)
     gdf["country"] = gdf.bus.str[:2]
     
+    # the .sjoin can lead to duplicates if a geom is in two regions
+    if gdf.index.duplicated().any():
+        import pycountry
+        # get all duplicated entries
+        duplicated_i = gdf.index[gdf.index.duplicated()]
+        # convert from raw data country name to iso-2-code
+        s = df.loc[duplicated_i, "Country"].apply(lambda x: pycountry.countries.lookup(x).alpha_2)
+        # Get a boolean mask where gdf's country column matches s's values for the same index
+        mask = gdf['country'] == gdf.index.map(s)
+        # Filter gdf using the mask
+        gdf_filtered = gdf[mask]
+        # concat not duplicated and filtered gdf
+        gdf = pd.concat([gdf.drop(duplicated_i), gdf_filtered]).sort_index()
+
     return gdf
 
 
@@ -115,7 +129,9 @@ def build_nodal_distribution_key(hotmaps, regions, countries):
         facilities = hotmaps.query("country == @country and Subsector == @sector")
 
         if not facilities.empty:
-            emissions = facilities["Emissions_ETS_2014"]
+            emissions = facilities["Emissions_ETS_2014"].fillna(
+                hotmaps["Emissions_EPRTR_2014"]
+            )
             if emissions.sum() == 0:
                 key = pd.Series(1 / len(facilities), facilities.index)
             else:
@@ -130,7 +146,7 @@ def build_nodal_distribution_key(hotmaps, regions, countries):
 
     return keys
 
-
+#%%
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
@@ -138,7 +154,7 @@ if __name__ == "__main__":
         snakemake = mock_snakemake(
             "build_industrial_distribution_key",
             simpl="",
-            clusters=48,
+            clusters=128,
         )
 
     logging.basicConfig(level=snakemake.config["logging"]["level"])

scripts/build_line_rating.py

@@ -0,0 +1,156 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2017-2020 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+# coding: utf-8
+"""
+Adds dynamic line rating timeseries to the base network.
+
+Relevant Settings
+-----------------
+
+.. code:: yaml
+
+    lines:
+        cutout:
+        line_rating:
+
+
+.. seealso::
+    Documentation of the configuration file ``config.yaml`
+Inputs
+------
+
+- ``data/cutouts``:
+- ``networks/base.nc``: confer :ref:`base`
+
+Outputs
+-------
+
+- ``resources/line_rating.nc``
+
+
+Description
+-----------
+
+The rule :mod:`build_line_rating` calculates the line rating for transmission lines.
+The line rating provides the maximal capacity of a transmission line considering the heat exchange with the environment.
+
+The following heat gains and losses are considered:
+
+- heat gain through resistive losses
+- heat gain through solar radiation
+- heat loss through radiation of the trasnmission line
+- heat loss through forced convection with wind
+- heat loss through natural convection
+
+
+With a heat balance considering the maximum temperature threshold of the transmission line,
+the maximal possible capacity factor "s_max_pu" for each transmission line at each time step is calculated.
+"""
+
+import logging
+import re
+
+import atlite
+import geopandas as gpd
+import numpy as np
+import pandas as pd
+import pypsa
+import xarray as xr
+from _helpers import configure_logging
+from shapely.geometry import LineString as Line
+from shapely.geometry import Point
+
+
+def calculate_resistance(T, R_ref, T_ref=293, alpha=0.00403):
+    """
+    Calculates the resistance at other temperatures than the reference
+    temperature.
+
+    Parameters
+    ----------
+    T : Temperature at which resistance is calculated in [°C] or [K]
+    R_ref : Resistance at reference temperature in [Ohm] or [Ohm/Per Length Unit]
+    T_ref : Reference temperature in [°C] or [K]
+    alpha: Temperature coefficient in [1/K]
+        Defaults are:
+            * T_ref : 20 °C
+            * alpha : 0.00403 1/K
+
+    Returns
+    -------
+    Resistance of at given temperature.
+    """
+    R = R_ref * (1 + alpha * (T - T_ref))
+    return R
+
+
+def calculate_line_rating(n, cutout):
+    """
+    Calculates the maximal allowed power flow in each line for each time step
+    considering the maximal temperature.
+
+    Parameters
+    ----------
+    n : pypsa.Network object containing information on grid
+
+    Returns
+    -------
+    xarray DataArray object with maximal power.
+    """
+    relevant_lines = n.lines[(n.lines["underground"] == False)]
+    buses = relevant_lines[["bus0", "bus1"]].values
+    x = n.buses.x
+    y = n.buses.y
+    shapes = [Line([Point(x[b0], y[b0]), Point(x[b1], y[b1])]) for (b0, b1) in buses]
+    shapes = gpd.GeoSeries(shapes, index=relevant_lines.index)
+    if relevant_lines.r_pu.eq(0).all():
+        # Overwrite standard line resistance with line resistance obtained from line type
+        r_per_length = n.line_types["r_per_length"]
+        R = (
+            relevant_lines.join(r_per_length, on=["type"])["r_per_length"] / 1000
+        )  # in meters
+        # If line type with bundles is given retrieve number of conductors per bundle
+        relevant_lines["n_bundle"] = (
+            relevant_lines["type"]
+            .where(relevant_lines["type"].str.contains("bundle"))
+            .dropna()
+            .apply(lambda x: int(re.findall(r"(\d+)-bundle", x)[0]))
+        )
+        # Set default number of bundles per line
+        relevant_lines["n_bundle"].fillna(1, inplace=True)
+        R *= relevant_lines["n_bundle"]
+        R = calculate_resistance(T=353, R_ref=R)
+    Imax = cutout.line_rating(shapes, R, D=0.0218, Ts=353, epsilon=0.8, alpha=0.8)
+    line_factor = relevant_lines.eval("v_nom * n_bundle * num_parallel") / 1e3  # in mW
+    da = xr.DataArray(
+        data=np.sqrt(3) * Imax * line_factor.values.reshape(-1, 1),
+        attrs=dict(
+            description="Maximal possible power in MW for given line considering line rating"
+        ),
+    )
+    return da
+
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake(
+            "build_line_rating",
+            network="elec",
+            simpl="",
+            clusters="5",
+            ll="v1.0",
+            opts="Co2L-4H",
+        )
+    configure_logging(snakemake)
+
+    n = pypsa.Network(snakemake.input.base_network)
+    time = pd.date_range(freq="h", **snakemake.config["snapshots"])
+    cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time)
+
+    da = calculate_line_rating(n, cutout)
+    da.to_netcdf(snakemake.output[0])

scripts/build_monthly_prices.py

@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Tue May 16 10:37:35 2023.
+
+This script extracts monthly fuel prices of oil, gas, coal and lignite,
+as well as CO2 prices
+
+
+Inputs
+------
+- ``data/energy-price-trends-xlsx-5619002.xlsx``: energy price index of fossil fuels
+- ``emission-spot-primary-market-auction-report-2019-data.xls``: CO2 Prices spot primary auction
+
+
+Outputs
+-------
+
+- ``data/validation/monthly_fuel_price.csv``
+- ``data/validation/CO2_price_2019.csv``
+
+
+Description
+-----------
+
+The rule :mod:`build_monthly_prices` collects monthly fuel prices and CO2 prices
+and translates them from different input sources to pypsa syntax
+
+Data sources:
+    [1] Fuel price index. Destatis
+    https://www.destatis.de/EN/Home/_node.html
+    [2] average annual fuel price lignite, ENTSO-E
+    https://2020.entsos-tyndp-scenarios.eu/fuel-commodities-and-carbon-prices/
+    [3] CO2 Prices, Emission spot primary auction, EEX
+    https://www.eex.com/en/market-data/environmental-markets/eua-primary-auction-spot-download
+
+
+Data was accessed at 16.5.2023
+"""
+
+import logging
+
+import pandas as pd
+from _helpers import configure_logging
+
+logger = logging.getLogger(__name__)
+
+
+# keywords in datasheet
+keywords = {
+    "coal": " GP09-051 Hard coal",
+    "lignite": " GP09-052 Lignite and lignite briquettes",
+    "oil": " GP09-0610 10 Mineral oil, crude",
+    "gas": "GP09-062 Natural gas",
+}
+
+# sheet names to pypsa syntax
+sheet_name_map = {
+    "coal": "5.1 Hard coal and lignite",
+    "lignite": "5.1 Hard coal and lignite",
+    "oil": "5.2 Mineral oil",
+    "gas": "5.3.1 Natural gas - indices",
+}
+
+
+# import fuel price 2015 in Eur/MWh
+# source lignite, price for 2020, scaled by price index, ENTSO-E [3]
+price_2020 = (
+    pd.Series({"coal": 3.0, "oil": 10.6, "gas": 5.6, "lignite": 1.1}) * 3.6
+)  # Eur/MWh
+
+# manual adjustment of coal price
+price_2020["coal"] = 2.4 * 3.6
+price_2020["lignite"] = 1.6 * 3.6
+
+
+def get_fuel_price():
+    price = {}
+    for carrier, keyword in keywords.items():
+        sheet_name = sheet_name_map[carrier]
+        df = pd.read_excel(
+            snakemake.input.fuel_price_raw,
+            sheet_name=sheet_name,
+            index_col=0,
+            skiprows=6,
+            nrows=18,
+        )
+        df = df.dropna(axis=0).iloc[:, :12]
+        start, end = df.index[0], str(int(df.index[-1][:4]) + 1)
+        df = df.stack()
+        df.index = pd.date_range(start=start, end=end, freq="MS", inclusive="left")
+        scale = price_2020[carrier] / df["2020"].mean()  # scale to 2020 price
+        df = df.mul(scale)
+        price[carrier] = df
+
+    return pd.concat(price, axis=1)
+
+
+def get_co2_price():
+    # emission price
+    co2_price = pd.read_excel(snakemake.input.co2_price_raw, index_col=1, header=5)
+    return co2_price["Auction Price €/tCO2"]
+
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake("build_monthly_prices")
+
+    configure_logging(snakemake)
+
+    fuel_price = get_fuel_price()
+    fuel_price.to_csv(snakemake.output.fuel_price)
+
+    co2_price = get_co2_price()
+    co2_price.to_csv(snakemake.output.co2_price)

scripts/build_natura_raster.py

@@ -54,6 +54,23 @@ logger = logging.getLogger(__name__)
 
 
 def determine_cutout_xXyY(cutout_name):
+    """
+    Determine the full extent of a cutout.
+
+    Since the coordinates of the cutout data are given as the
+    center of the grid cells, the extent of the cutout is
+    calculated by adding/subtracting half of the grid cell size.
+
+
+    Parameters
+    ----------
+    cutout_name : str
+        Path to the cutout.
+
+    Returns
+    -------
+    A list of extent coordinates in the order [x, X, y, Y].
+    """
     cutout = atlite.Cutout(cutout_name)
     assert cutout.crs.to_epsg() == 4326
     x, X, y, Y = cutout.extent

scripts/build_powerplants.py

@@ -98,13 +98,15 @@ def add_custom_powerplants(ppl, custom_powerplants, custom_ppl_query=False):
 
 
 def replace_natural_gas_technology(df):
-    mapping = {"Steam Turbine": "OCGT", "Combustion Engine": "OCGT"}
-    tech = df.Technology.replace(mapping).fillna("OCGT")
-    return df.Technology.where(df.Fueltype != "Natural Gas", tech)
+    mapping = {"Steam Turbine": "CCGT", "Combustion Engine": "OCGT"}
+    tech = df.Technology.replace(mapping).fillna("CCGT")
+    return df.Technology.mask(df.Fueltype == "Natural Gas", tech)
 
 
 def replace_natural_gas_fueltype(df):
-    return df.Fueltype.where(df.Fueltype != "Natural Gas", df.Technology)
+    return df.Fueltype.mask(
+        (df.Technology == "OCGT") | (df.Technology == "CCGT"), "Natural Gas"
+    )
 
 
 if __name__ == "__main__":

scripts/build_renewable_profiles.py

@@ -186,6 +186,7 @@ import time
 import atlite
 import geopandas as gpd
 import numpy as np
+import pandas as pd
 import xarray as xr
 from _helpers import configure_logging
 from dask.distributed import Client
@@ -222,7 +223,8 @@ if __name__ == "__main__":
     else:
         client = None
 
-    cutout = atlite.Cutout(snakemake.input.cutout)
+    sns = pd.date_range(freq="h", **snakemake.config["snapshots"])
+    cutout = atlite.Cutout(snakemake.input.cutout).sel(time=sns)
     regions = gpd.read_file(snakemake.input.regions)
     assert not regions.empty, (
         f"List of regions in {snakemake.input.regions} is empty, please "
@@ -372,4 +374,6 @@ if __name__ == "__main__":
         ds["profile"] = ds["profile"].where(ds["profile"] >= min_p_max_pu, 0)
 
     ds.to_netcdf(snakemake.output.profile)
-    client.shutdown()
+
+    if client is not None:
+        client.shutdown()

scripts/build_sequestration_potentials.py

@@ -28,9 +28,7 @@ def allocate_sequestration_potential(
     overlay["share"] = area(overlay) / overlay["area_sqkm"]
     adjust_cols = overlay.columns.difference({"name", "area_sqkm", "geometry", "share"})
     overlay[adjust_cols] = overlay[adjust_cols].multiply(overlay["share"], axis=0)
-    gdf_regions = overlay.groupby("name").sum()
-    gdf_regions.drop(["area_sqkm", "share"], axis=1, inplace=True)
-    return gdf_regions.squeeze()
+    return overlay.dissolve("name", aggfunc="sum")[attr]
 
 
 if __name__ == "__main__":

scripts/cluster_network.py

@@ -461,7 +461,7 @@ if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
 
-        snakemake = mock_snakemake("cluster_network", simpl="", clusters="37c")
+        snakemake = mock_snakemake("cluster_network", simpl="", clusters="37")
     configure_logging(snakemake)
 
     params = snakemake.params
@@ -483,6 +483,23 @@ if __name__ == "__main__":
     else:
         n_clusters = int(snakemake.wildcards.clusters)
 
+    if params.cluster_network.get("consider_efficiency_classes", False):
+        carriers = []
+        for c in aggregate_carriers:
+            gens = n.generators.query("carrier == @c")
+            low = gens.efficiency.quantile(0.10)
+            high = gens.efficiency.quantile(0.90)
+            if low >= high:
+                carriers += [c]
+            else:
+                labels = ["low", "medium", "high"]
+                suffix = pd.cut(
+                    gens.efficiency, bins=[0, low, high, 1], labels=labels
+                ).astype(str)
+                carriers += [f"{c} {label} efficiency" for label in labels]
+                n.generators.carrier.update(gens.carrier + " " + suffix + " efficiency")
+        aggregate_carriers = carriers
+
     if n_clusters == len(n.buses):
         # Fast-path if no clustering is necessary
         busmap = n.buses.index.to_series()
@@ -524,6 +541,11 @@ if __name__ == "__main__":
 
     update_p_nom_max(clustering.network)
 
+    if params.cluster_network.get("consider_efficiency_classes"):
+        labels = [f" {label} efficiency" for label in ["low", "medium", "high"]]
+        nc = clustering.network
+        nc.generators["carrier"] = nc.generators.carrier.replace(labels, "", regex=True)
+
     clustering.network.meta = dict(
         snakemake.config, **dict(wildcards=dict(snakemake.wildcards))
     )

scripts/copy_config.py

@@ -11,25 +11,13 @@ from shutil import copy
 
 import yaml
 
-files = {
-    "config/config.yaml": "config.yaml",
-    "Snakefile": "Snakefile",
-    "scripts/solve_network.py": "solve_network.py",
-    "scripts/prepare_sector_network.py": "prepare_sector_network.py",
-}
-
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
 
         snakemake = mock_snakemake("copy_config")
 
-    basepath = Path(f"results/{snakemake.params.RDIR}config/")
-
-    for f, name in files.items():
-        copy(f, basepath / name)
-
-    with open(basepath / "config.snakemake.yaml", "w") as yaml_file:
+    with open(snakemake.output[0], "w") as yaml_file:
         yaml.dump(
             snakemake.config,
             yaml_file,

scripts/make_summary.py

@@ -16,7 +16,6 @@ import sys
 import numpy as np
 import pandas as pd
 import pypsa
-from _helpers import override_component_attrs
 from prepare_sector_network import prepare_costs
 
 idx = pd.IndexSlice
@@ -300,9 +299,9 @@ def calculate_energy(n, label, energy):
                 )
                 # remove values where bus is missing (bug in nomopyomo)
                 no_bus = c.df.index[c.df["bus" + port] == ""]
-                totals.loc[no_bus] = n.component_attrs[c.name].loc[
-                    "p" + port, "default"
-                ]
+                totals.loc[no_bus] = float(
+                    n.component_attrs[c.name].loc["p" + port, "default"]
+                )
                 c_energies -= totals.groupby(c.df.carrier).sum()
 
         c_energies = pd.concat([c_energies], keys=[c.list_name])
@@ -659,8 +658,7 @@ def make_summaries(networks_dict):
     for label, filename in networks_dict.items():
         logger.info(f"Make summary for scenario {label}, using {filename}")
 
-        overrides = override_component_attrs(snakemake.input.overrides)
-        n = pypsa.Network(filename, override_component_attrs=overrides)
+        n = pypsa.Network(filename)
 
         assign_carriers(n)
         assign_locations(n)

scripts/plot_network.py

@@ -20,7 +20,6 @@ import geopandas as gpd
 import matplotlib.pyplot as plt
 import pandas as pd
 import pypsa
-from _helpers import override_component_attrs
 from make_summary import assign_carriers
 from plot_summary import preferred_order, rename_techs
 from pypsa.plot import add_legend_circles, add_legend_lines, add_legend_patches
@@ -930,8 +929,7 @@ if __name__ == "__main__":
 
     logging.basicConfig(level=snakemake.config["logging"]["level"])
 
-    overrides = override_component_attrs(snakemake.input.overrides)
-    n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
+    n = pypsa.Network(snakemake.input.network)
 
     regions = gpd.read_file(snakemake.input.regions).set_index("name")
 

scripts/plot_statistics.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+import matplotlib.pyplot as plt
+import pypsa
+import seaborn as sns
+from _helpers import configure_logging
+
+sns.set_theme("paper", style="whitegrid")
+
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake(
+            "plot_elec_statistics",
+            simpl="",
+            opts="Ept-12h",
+            clusters="37",
+            ll="v1.0",
+        )
+    configure_logging(snakemake)
+
+    n = pypsa.Network(snakemake.input.network)
+
+    n.loads.carrier = "load"
+    n.carriers.loc["load", ["nice_name", "color"]] = "Load", "darkred"
+    colors = n.carriers.set_index("nice_name").color.where(
+        lambda s: s != "", "lightgrey"
+    )
+
+    # %%
+
+    def rename_index(ds):
+        specific = ds.index.map(lambda x: f"{x[1]}\n({x[0]})")
+        generic = ds.index.get_level_values("carrier")
+        duplicated = generic.duplicated(keep=False)
+        index = specific.where(duplicated, generic)
+        return ds.set_axis(index)
+
+    def plot_static_per_carrier(ds, ax, drop_zero=True):
+        if drop_zero:
+            ds = ds[ds != 0]
+        ds = ds.dropna()
+        c = colors[ds.index.get_level_values("carrier")]
+        ds = ds.pipe(rename_index)
+        label = f"{ds.attrs['name']} [{ds.attrs['unit']}]"
+        ds.plot.barh(color=c.values, xlabel=label, ax=ax)
+        ax.grid(axis="y")
+
+    fig, ax = plt.subplots()
+    ds = n.statistics.capacity_factor().dropna()
+    plot_static_per_carrier(ds, ax)
+    fig.savefig(snakemake.output.capacity_factor_bar)
+
+    fig, ax = plt.subplots()
+    ds = n.statistics.installed_capacity().dropna()
+    ds = ds.drop("Line")
+    ds = ds.drop(("Generator", "Load"))
+    ds = ds / 1e3
+    ds.attrs["unit"] = "GW"
+    plot_static_per_carrier(ds, ax)
+    fig.savefig(snakemake.output.installed_capacity_bar)
+
+    fig, ax = plt.subplots()
+    ds = n.statistics.optimal_capacity()
+    ds = ds.drop("Line")
+    ds = ds.drop(("Generator", "Load"))
+    ds = ds / 1e3
+    ds.attrs["unit"] = "GW"
+    plot_static_per_carrier(ds, ax)
+    fig.savefig(snakemake.output.optimal_capacity_bar)
+
+    fig, ax = plt.subplots()
+    ds = n.statistics.capex()
+    plot_static_per_carrier(ds, ax)
+    fig.savefig(snakemake.output.capital_expenditure_bar)
+
+    fig, ax = plt.subplots()
+    ds = n.statistics.opex()
+    plot_static_per_carrier(ds, ax)
+    fig.savefig(snakemake.output.operational_expenditure_bar)
+
+    fig, ax = plt.subplots()
+    ds = n.statistics.curtailment()
+    plot_static_per_carrier(ds, ax)
+    fig.savefig(snakemake.output.curtailment_bar)
+
+    fig, ax = plt.subplots()
+    ds = n.statistics.supply()
+    ds = ds.drop("Line")
+    ds = ds / 1e6
+    ds.attrs["unit"] = "TWh"
+    plot_static_per_carrier(ds, ax)
+    fig.savefig(snakemake.output.supply_bar)
+
+    fig, ax = plt.subplots()
+    ds = n.statistics.withdrawal()
+    ds = ds.drop("Line")
+    ds = ds / -1e6
+    ds.attrs["unit"] = "TWh"
+    plot_static_per_carrier(ds, ax)
+    fig.savefig(snakemake.output.withdrawal_bar)
+
+    fig, ax = plt.subplots()
+    ds = n.statistics.market_value()
+    plot_static_per_carrier(ds, ax)
+    fig.savefig(snakemake.output.market_value_bar)
+
+    # touch file
+    with open(snakemake.output.barplots_touch, "a"):
+        pass

scripts/plot_validation_cross_border_flows.py

@@ -0,0 +1,242 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+import country_converter as coco
+import matplotlib.pyplot as plt
+import pandas as pd
+import pypsa
+import seaborn as sns
+from _helpers import configure_logging
+
+sns.set_theme("paper", style="whitegrid")
+
+cc = coco.CountryConverter()
+
+color_country = {
+    "AL": "#440154",
+    "AT": "#482677",
+    "BA": "#43398e",
+    "BE": "#3953a4",
+    "BG": "#2c728e",
+    "CH": "#228b8d",
+    "CZ": "#1f9d8a",
+    "DE": "#29af7f",
+    "DK": "#3fbc73",
+    "EE": "#5ec962",
+    "ES": "#84d44b",
+    "FI": "#addc30",
+    "FR": "#d8e219",
+    "GB": "#fde725",
+    "GR": "#f0f921",
+    "HR": "#f1c25e",
+    "HU": "#f4a784",
+    "IE": "#f78f98",
+    "IT": "#f87ea0",
+    "LT": "#f87a9a",
+    "LU": "#f57694",
+    "LV": "#f3758d",
+    "ME": "#f37685",
+    "MK": "#f37b7c",
+    "NL": "#FF6666",
+    "NO": "#FF3333",
+    "PL": "#eb0000",
+    "PT": "#d70000",
+    "RO": "#c00000",
+    "RS": "#a50000",
+    "SE": "#8a0000",
+    "SI": "#6f0000",
+    "SK": "#550000",
+}
+
+
+def sort_one_country(country, df):
+    indices = [link for link in df.columns if country in link]
+    df_country = df[indices].copy()
+    for link in df_country.columns:
+        if country in link[5:]:
+            df_country[link] = -df_country[link]
+            link_reverse = str(link[5:] + " - " + link[:2])
+            df_country = df_country.rename(columns={link: link_reverse})
+
+    return df_country.reindex(sorted(df_country.columns), axis=1)
+
+
+def cross_border_time_series(countries, data):
+    fig, ax = plt.subplots(2 * len(countries), 1, figsize=(15, 10 * len(countries)))
+    axis = 0
+
+    for country in countries:
+        ymin = 0
+        ymax = 0
+        for df in data:
+            df_country = sort_one_country(country, df)
+            df_neg, df_pos = df_country.clip(upper=0), df_country.clip(lower=0)
+
+            color = [color_country[link[5:]] for link in df_country.columns]
+
+            df_pos.plot.area(
+                ax=ax[axis], stacked=True, linewidth=0.0, color=color, ylim=[-1, 1]
+            )
+
+            df_neg.plot.area(
+                ax=ax[axis], stacked=True, linewidth=0.0, color=color, ylim=[-1, 1]
+            )
+            if (axis % 2) == 0:
+                title = "Historic"
+            else:
+                title = "Optimized"
+
+            ax[axis].set_title(
+                title + " Import / Export for " + cc.convert(country, to="name_short")
+            )
+
+            # Custom legend elements
+            legend_elements = []
+
+            for link in df_country.columns:
+                legend_elements = legend_elements + [
+                    plt.fill_between(
+                        [],
+                        [],
+                        color=color_country[link[5:]],
+                        label=cc.convert(link[5:], to="name_short"),
+                    )
+                ]
+
+            # Create the legend
+            ax[axis].legend(handles=legend_elements, loc="upper right")
+
+            # rescale the y axis
+            neg_min = df_neg.sum(axis=1).min() * 1.2
+            if neg_min < ymin:
+                ymin = neg_min
+
+            pos_max = df_pos.sum(axis=1).max() * 1.2
+            if pos_max < ymax:
+                ymax = pos_max
+
+            axis = axis + 1
+
+        for x in range(axis - 2, axis):
+            ax[x].set_ylim([neg_min, pos_max])
+
+    fig.savefig(snakemake.output.trade_time_series, bbox_inches="tight")
+
+
+def cross_border_bar(countries, data):
+    df_positive = pd.DataFrame()
+    df_negative = pd.DataFrame()
+    color = []
+
+    for country in countries:
+        order = 0
+        for df in data:
+            df_country = sort_one_country(country, df)
+            df_neg, df_pos = df_country.clip(upper=0), df_country.clip(lower=0)
+
+            if (order % 2) == 0:
+                title = "Historic"
+            else:
+                title = "Optimized"
+
+            df_positive_new = pd.DataFrame(data=df_pos.sum()).T.rename(
+                {0: title + " " + cc.convert(country, to="name_short")}
+            )
+            df_negative_new = pd.DataFrame(data=df_neg.sum()).T.rename(
+                {0: title + " " + cc.convert(country, to="name_short")}
+            )
+
+            df_positive = pd.concat([df_positive_new, df_positive])
+            df_negative = pd.concat([df_negative_new, df_negative])
+
+            order = order + 1
+
+    color = [color_country[link[5:]] for link in df_positive.columns]
+
+    fig, ax = plt.subplots(figsize=(15, 60))
+
+    df_positive.plot.barh(ax=ax, stacked=True, color=color, zorder=2)
+    df_negative.plot.barh(ax=ax, stacked=True, color=color, zorder=2)
+
+    plt.grid(axis="x", zorder=0)
+    plt.grid(axis="y", zorder=0)
+
+    # Custom legend elements
+    legend_elements = []
+
+    for country in list(color_country.keys()):
+        legend_elements = legend_elements + [
+            plt.fill_between(
+                [],
+                [],
+                color=color_country[country],
+                label=cc.convert(country, to="name_short"),
+            )
+        ]
+
+    # Create the legend
+    plt.legend(handles=legend_elements, loc="upper right")
+
+    fig.savefig(snakemake.output.cross_border_bar, bbox_inches="tight")
+
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake(
+            "plot_electricity_prices",
+            simpl="",
+            opts="Ept-12h",
+            clusters="37",
+            ll="v1.0",
+        )
+    configure_logging(snakemake)
+
+    countries = snakemake.params.countries
+
+    n = pypsa.Network(snakemake.input.network)
+    n.loads.carrier = "load"
+
+    historic = pd.read_csv(
+        snakemake.input.cross_border_flows,
+        index_col=0,
+        header=0,
+        parse_dates=True,
+    )
+
+    if len(historic.index) > len(n.snapshots):
+        historic = historic.resample(n.snapshots.inferred_freq).mean().loc[n.snapshots]
+
+    # Preparing network data to be shaped similar to ENTSOE datastructure
+    optimized_links = n.links_t.p0.rename(
+        columns=dict(n.links.bus0.str[:2] + " - " + n.links.bus1.str[:2])
+    )
+    optimized_lines = n.lines_t.p0.rename(
+        columns=dict(n.lines.bus0.str[:2] + " - " + n.lines.bus1.str[:2])
+    )
+    optimized = pd.concat([optimized_links, optimized_lines], axis=1)
+
+    # Drop internal country connection
+    optimized.drop(
+        [c for c in optimized.columns if c[:2] == c[5:]], axis=1, inplace=True
+    )
+
+    # align columns name
+    for c1 in optimized.columns:
+        for c2 in optimized.columns:
+            if c1[:2] == c2[5:] and c2[:2] == c1[5:]:
+                optimized = optimized.rename(columns={c1: c2})
+
+    optimized = optimized.groupby(lambda x: x, axis=1).sum()
+
+    cross_border_bar(countries, [historic, optimized])
+
+    cross_border_time_series(countries, [historic, optimized])
+
+    # touch file
+    with open(snakemake.output.plots_touch, "a"):
+        pass

scripts/plot_validation_electricity_prices.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+import matplotlib.pyplot as plt
+import pandas as pd
+import pypsa
+import seaborn as sns
+from _helpers import configure_logging
+from pypsa.statistics import get_bus_and_carrier
+
+sns.set_theme("paper", style="whitegrid")
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake(
+            "plot_electricity_prices",
+            simpl="",
+            opts="Ept-12h",
+            clusters="37",
+            ll="v1.0",
+        )
+    configure_logging(snakemake)
+
+    n = pypsa.Network(snakemake.input.network)
+    n.loads.carrier = "load"
+
+    historic = pd.read_csv(
+        snakemake.input.electricity_prices,
+        index_col=0,
+        header=0,
+        parse_dates=True,
+    )
+
+    if len(historic.index) > len(n.snapshots):
+        historic = historic.resample(n.snapshots.inferred_freq).mean().loc[n.snapshots]
+
+    optimized = n.buses_t.marginal_price.groupby(n.buses.country, axis=1).mean()
+
+    data = pd.concat([historic, optimized], keys=["Historic", "Optimized"], axis=1)
+    data.columns.names = ["Kind", "Country"]
+
+    fig, ax = plt.subplots(figsize=(6, 6))
+
+    df = data.mean().unstack().T
+    df.plot.barh(ax=ax, xlabel="Electricity Price [€/MWh]", ylabel="")
+    ax.grid(axis="y")
+    fig.savefig(snakemake.output.price_bar, bbox_inches="tight")
+
+    fig, ax = plt.subplots()
+
+    df = data.groupby(level="Kind", axis=1).mean()
+    df.plot(ax=ax, xlabel="", ylabel="Electricity Price [€/MWh]", alpha=0.8)
+    ax.grid(axis="x")
+    fig.savefig(snakemake.output.price_line, bbox_inches="tight")
+
+    # touch file
+    with open(snakemake.output.plots_touch, "a"):
+        pass

scripts/plot_validation_electricity_production.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+import matplotlib.pyplot as plt
+import pandas as pd
+import pypsa
+import seaborn as sns
+from _helpers import configure_logging
+from pypsa.statistics import get_bus_and_carrier
+
+sns.set_theme("paper", style="whitegrid")
+
+carrier_groups = {
+    "Offshore Wind (AC)": "Offshore Wind",
+    "Offshore Wind (DC)": "Offshore Wind",
+    "Open-Cycle Gas": "Gas",
+    "Combined-Cycle Gas": "Gas",
+    "Reservoir & Dam": "Hydro",
+    "Pumped Hydro Storage": "Hydro",
+}
+
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake(
+            "plot_validation_electricity_production",
+            simpl="",
+            opts="Ept",
+            clusters="37c",
+            ll="v1.0",
+        )
+    configure_logging(snakemake)
+
+    n = pypsa.Network(snakemake.input.network)
+    n.loads.carrier = "load"
+
+    historic = pd.read_csv(
+        snakemake.input.electricity_production,
+        index_col=0,
+        header=[0, 1],
+        parse_dates=True,
+    )
+
+    colors = n.carriers.set_index("nice_name").color.where(
+        lambda s: s != "", "lightgrey"
+    )
+    colors["Offshore Wind"] = colors["Offshore Wind (AC)"]
+    colors["Gas"] = colors["Combined-Cycle Gas"]
+    colors["Hydro"] = colors["Reservoir & Dam"]
+    colors["Other"] = "lightgray"
+
+    if len(historic.index) > len(n.snapshots):
+        historic = historic.resample(n.snapshots.inferred_freq).mean().loc[n.snapshots]
+
+    optimized = n.statistics.dispatch(
+        groupby=get_bus_and_carrier, aggregate_time=False
+    ).T
+    optimized = optimized[["Generator", "StorageUnit"]].droplevel(0, axis=1)
+    optimized = optimized.rename(columns=n.buses.country, level=0)
+    optimized = optimized.rename(columns=carrier_groups, level=1)
+    optimized = optimized.groupby(axis=1, level=[0, 1]).sum()
+
+    data = pd.concat([historic, optimized], keys=["Historic", "Optimized"], axis=1)
+    data.columns.names = ["Kind", "Country", "Carrier"]
+    data = data.mul(n.snapshot_weightings.generators, axis=0)
+
+    # total production per carrier
+    fig, ax = plt.subplots(figsize=(6, 6))
+
+    df = data.groupby(level=["Kind", "Carrier"], axis=1).sum().sum().unstack().T
+    df = df / 1e6  # TWh
+    df.plot.barh(ax=ax, xlabel="Electricity Production [TWh]", ylabel="")
+    ax.grid(axis="y")
+    fig.savefig(snakemake.output.production_bar, bbox_inches="tight")
+
+    # highest diffs
+
+    fig, ax = plt.subplots(figsize=(6, 10))
+
+    df = data.sum() / 1e6  # TWh
+    df = df["Optimized"] - df["Historic"]
+    df = df.dropna().sort_values()
+    df = pd.concat([df.iloc[:5], df.iloc[-5:]])
+    c = colors[df.index.get_level_values(1)]
+    df.plot.barh(
+        xlabel="Optimized Production - Historic Production [TWh]", ax=ax, color=c.values
+    )
+    ax.set_title("Strongest Deviations")
+    ax.grid(axis="y")
+    fig.savefig(snakemake.output.production_deviation_bar, bbox_inches="tight")
+
+    # seasonal operation
+
+    fig, axes = plt.subplots(3, 1, figsize=(9, 9))
+
+    df = (
+        data.groupby(level=["Kind", "Carrier"], axis=1)
+        .sum()
+        .resample("1W")
+        .mean()
+        .clip(lower=0)
+    )
+    df = df / 1e3
+
+    order = (
+        (df["Historic"].diff().abs().sum() / df["Historic"].sum()).sort_values().index
+    )
+    c = colors[order]
+    optimized = df["Optimized"].reindex(order, axis=1, level=1)
+    historical = df["Historic"].reindex(order, axis=1, level=1)
+
+    kwargs = dict(color=c, legend=False, ylabel="Production [GW]", xlabel="")
+
+    optimized.plot.area(ax=axes[0], **kwargs, title="Optimized")
+    historical.plot.area(ax=axes[1], **kwargs, title="Historic")
+
+    diff = optimized - historical
+    diff.clip(lower=0).plot.area(
+        ax=axes[2], **kwargs, title="$\Delta$ (Optimized - Historic)"
+    )
+    lim = axes[2].get_ylim()[1]
+    diff.clip(upper=0).plot.area(ax=axes[2], **kwargs)
+    axes[2].set_ylim(bottom=-lim, top=lim)
+
+    h, l = axes[0].get_legend_handles_labels()
+    fig.legend(
+        h[::-1],
+        l[::-1],
+        loc="center left",
+        bbox_to_anchor=(1, 0.5),
+        ncol=1,
+        frameon=False,
+        labelspacing=1,
+    )
+    fig.savefig(snakemake.output.seasonal_operation_area, bbox_inches="tight")
+
+    # touch file
+    with open(snakemake.output.plots_touch, "a"):
+        pass

scripts/prepare_network.py

@@ -65,6 +65,7 @@ import pandas as pd
 import pypsa
 from _helpers import configure_logging
 from add_electricity import load_costs, update_transmission_costs
+from pypsa.descriptors import expand_series
 
 idx = pd.IndexSlice
 
@@ -103,10 +104,30 @@ def add_emission_prices(n, emission_prices={"co2": 0.0}, exclude_co2=False):
     ).sum(axis=1)
     gen_ep = n.generators.carrier.map(ep) / n.generators.efficiency
     n.generators["marginal_cost"] += gen_ep
+    n.generators_t["marginal_cost"] += gen_ep[n.generators_t["marginal_cost"].columns]
     su_ep = n.storage_units.carrier.map(ep) / n.storage_units.efficiency_dispatch
     n.storage_units["marginal_cost"] += su_ep
 
 
+def add_dynamic_emission_prices(n):
+    co2_price = pd.read_csv(snakemake.input.co2_price, index_col=0, parse_dates=True)
+    co2_price = co2_price[~co2_price.index.duplicated()]
+    co2_price = (
+        co2_price.reindex(n.snapshots).fillna(method="ffill").fillna(method="bfill")
+    )
+
+    emissions = (
+        n.generators.carrier.map(n.carriers.co2_emissions) / n.generators.efficiency
+    )
+    co2_cost = expand_series(emissions, n.snapshots).T.mul(co2_price.iloc[:, 0], axis=0)
+
+    static = n.generators.marginal_cost
+    dynamic = n.get_switchable_as_dense("Generator", "marginal_cost")
+
+    marginal_cost = dynamic + co2_cost.reindex(columns=dynamic.columns, fill_value=0)
+    n.generators_t.marginal_cost = marginal_cost.loc[:, marginal_cost.ne(static).any()]
+
+
 def set_line_s_max_pu(n, s_max_pu=0.7):
     n.lines["s_max_pu"] = s_max_pu
     logger.info(f"N-1 security margin of lines set to {s_max_pu}")
@@ -253,12 +274,13 @@ def set_line_nom_max(
     n.links.p_nom_max.clip(upper=p_nom_max_set, inplace=True)
 
 
+# %%
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
 
         snakemake = mock_snakemake(
-            "prepare_network", simpl="", clusters="40", ll="v0.3", opts="Co2L-24H"
+            "prepare_network", simpl="", clusters="37", ll="v1.0", opts="Ept"
         )
     configure_logging(snakemake)
 
@@ -332,7 +354,12 @@ if __name__ == "__main__":
                     c.df.loc[sel, attr] *= factor
 
     for o in opts:
-        if "Ep" in o:
+        if "Ept" in o:
+            logger.info(
+                "Setting time dependent emission prices according spot market price"
+            )
+            add_dynamic_emission_prices(n)
+        elif "Ep" in o:
             m = re.findall("[0-9]*\.?[0-9]+$", o)
             if len(m) > 0:
                 logger.info("Setting emission prices according to wildcard value.")

scripts/prepare_sector_network.py

@@ -17,11 +17,7 @@ import numpy as np
 import pandas as pd
 import pypsa
 import xarray as xr
-from _helpers import (
-    generate_periodic_profiles,
-    override_component_attrs,
-    update_config_with_sector_opts,
-)
+from _helpers import generate_periodic_profiles, update_config_with_sector_opts
 from add_electricity import calculate_annuity, sanitize_carriers
 from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2
 from networkx.algorithms import complement
@@ -2163,12 +2159,11 @@ def add_biomass(n, costs):
     )
 
     if options["biomass_transport"]:
-        transport_costs = pd.read_csv(
-            snakemake.input.biomass_transport_costs,
-            index_col=0,
-        ).squeeze()
-
         # add biomass transport
+        transport_costs = pd.read_csv(
+            snakemake.input.biomass_transport_costs, index_col=0
+        )
+        transport_costs = transport_costs.squeeze()
         biomass_transport = create_network_topology(
             n, "biomass transport ", bidirectional=False
         )
@@ -2192,6 +2187,27 @@ def add_biomass(n, costs):
             carrier="solid biomass transport",
         )
 
+    elif options["biomass_spatial"]:
+        # add artificial biomass generators at nodes which include transport costs
+        transport_costs = pd.read_csv(
+            snakemake.input.biomass_transport_costs, index_col=0
+        )
+        transport_costs = transport_costs.squeeze()
+        bus_transport_costs = spatial.biomass.nodes.to_series().apply(
+            lambda x: transport_costs[x[:2]]
+        )
+        average_distance = 200  # km #TODO: validate this assumption
+
+        n.madd(
+            "Generator",
+            spatial.biomass.nodes,
+            bus=spatial.biomass.nodes,
+            carrier="solid biomass",
+            p_nom=10000,
+            marginal_cost=costs.at["solid biomass", "fuel"]
+            + bus_transport_costs * average_distance,
+        )
+
     # AC buses with district heating
     urban_central = n.buses.index[n.buses.carrier == "urban central heat"]
     if not urban_central.empty and options["chp"]:
@@ -3283,8 +3299,7 @@ if __name__ == "__main__":
 
     investment_year = int(snakemake.wildcards.planning_horizons[-4:])
 
-    overrides = override_component_attrs(snakemake.input.overrides)
-    n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
+    n = pypsa.Network(snakemake.input.network)
 
     pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0)
     nhours = n.snapshot_weightings.generators.sum()

scripts/retrieve_monthly_fuel_prices.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+"""
+Retrieve monthly fuel prices from Destatis.
+"""
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+from pathlib import Path
+
+from _helpers import configure_logging, progress_retrieve
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake("retrieve_monthly_fuel_prices")
+        rootpath = ".."
+    else:
+        rootpath = "."
+    configure_logging(snakemake)
+
+    url = "https://www.destatis.de/EN/Themes/Economy/Prices/Publications/Downloads-Energy-Price-Trends/energy-price-trends-xlsx-5619002.xlsx?__blob=publicationFile"
+
+    to_fn = Path(rootpath) / Path(snakemake.output[0])
+
+    logger.info(f"Downloading monthly fuel prices from '{url}'.")
+    disable_progress = snakemake.config["run"].get("disable_progressbar", False)
+    progress_retrieve(url, to_fn, disable=disable_progress)
+
+    logger.info(f"Monthly fuel prices available at {to_fn}")

scripts/simplify_network.py

@@ -613,6 +613,7 @@ if __name__ == "__main__":
         "substation_lv",
         "substation_off",
         "geometry",
+        "underground",
     ]
     n.buses.drop(remove, axis=1, inplace=True, errors="ignore")
     n.lines.drop(remove, axis=1, errors="ignore", inplace=True)

scripts/solve_network.py

@@ -33,11 +33,7 @@ import numpy as np
 import pandas as pd
 import pypsa
 import xarray as xr
-from _helpers import (
-    configure_logging,
-    override_component_attrs,
-    update_config_with_sector_opts,
-)
+from _helpers import configure_logging, update_config_with_sector_opts
 
 logger = logging.getLogger(__name__)
 pypsa.pf.logger.setLevel(logging.WARNING)
@@ -152,7 +148,7 @@ def prepare_network(
     if "clip_p_max_pu" in solve_opts:
         for df in (
             n.generators_t.p_max_pu,
-            n.generators_t.p_min_pu,  # TODO: check if this can be removed
+            n.generators_t.p_min_pu,
             n.storage_units_t.inflow,
         ):
             df.where(df > solve_opts["clip_p_max_pu"], other=0.0, inplace=True)
@@ -280,13 +276,13 @@ def add_EQ_constraints(n, o, scaling=1e-1):
     float_regex = "[0-9]*\.?[0-9]+"
     level = float(re.findall(float_regex, o)[0])
     if o[-1] == "c":
-        ggrouper = n.generators.bus.map(n.buses.country).to_xarray()
-        lgrouper = n.loads.bus.map(n.buses.country).to_xarray()
-        sgrouper = n.storage_units.bus.map(n.buses.country).to_xarray()
+        ggrouper = n.generators.bus.map(n.buses.country)
+        lgrouper = n.loads.bus.map(n.buses.country)
+        sgrouper = n.storage_units.bus.map(n.buses.country)
     else:
-        ggrouper = n.generators.bus.to_xarray()
-        lgrouper = n.loads.bus.to_xarray()
-        sgrouper = n.storage_units.bus.to_xarray()
+        ggrouper = n.generators.bus
+        lgrouper = n.loads.bus
+        sgrouper = n.storage_units.bus
     load = (
         n.snapshot_weightings.generators
         @ n.loads_t.p_set.groupby(lgrouper, axis=1).sum()
@@ -300,7 +296,7 @@ def add_EQ_constraints(n, o, scaling=1e-1):
     p = n.model["Generator-p"]
     lhs_gen = (
         (p * (n.snapshot_weightings.generators * scaling))
-        .groupby(ggrouper)
+        .groupby(ggrouper.to_xarray())
         .sum()
         .sum("snapshot")
     )
@@ -309,7 +305,7 @@ def add_EQ_constraints(n, o, scaling=1e-1):
         spillage = n.model["StorageUnit-spill"]
         lhs_spill = (
             (spillage * (-n.snapshot_weightings.stores * scaling))
-            .groupby(sgrouper)
+            .groupby(sgrouper.to_xarray())
             .sum()
             .sum("snapshot")
         )
@@ -599,47 +595,46 @@ def extra_functionality(n, snapshots):
 
 def solve_network(n, config, solving, opts="", **kwargs):
     set_of_options = solving["solver"]["options"]
-    solver_options = solving["solver_options"][set_of_options] if set_of_options else {}
-    solver_name = solving["solver"]["name"]
     cf_solving = solving["options"]
-    track_iterations = cf_solving.get("track_iterations", False)
-    min_iterations = cf_solving.get("min_iterations", 4)
-    max_iterations = cf_solving.get("max_iterations", 6)
-    transmission_losses = cf_solving.get("transmission_losses", 0)
-    multi_horizon = True if config["foresight"] == "perfect" else False
+   
+    kwargs["multi_investment_periods"] = True if config["foresight"] == "perfect" else False
+    kwargs["solver_options"] = (
+        solving["solver_options"][set_of_options] if set_of_options else {}
+    )
+    kwargs["solver_name"] = solving["solver"]["name"]
+    kwargs["extra_functionality"] = extra_functionality
+    kwargs["transmission_losses"] = cf_solving.get("transmission_losses", False)
+    kwargs["linearized_unit_commitment"] = cf_solving.get(
+        "linearized_unit_commitment", False
+    )
+    kwargs["assign_all_duals"] = cf_solving.get("assign_all_duals", False)
+
+    rolling_horizon = cf_solving.pop("rolling_horizon", False)
+    skip_iterations = cf_solving.pop("skip_iterations", False)
+    if not n.lines.s_nom_extendable.any():
+        skip_iterations = True
+        logger.info("No expandable lines found. Skipping iterative solving.")
 
     # add to network for extra_functionality
     n.config = config
     n.opts = opts
 
-    skip_iterations = cf_solving.get("skip_iterations", False)
-    if not n.lines.s_nom_extendable.any():
-        skip_iterations = True
-        logger.info("No expandable lines found. Skipping iterative solving.")
-
-    if skip_iterations:
-        status, condition = n.optimize(
-            solver_name=solver_name,
-            transmission_losses=transmission_losses,
-            extra_functionality=extra_functionality,
-            multi_investment_periods=multi_horizon,
-            **solver_options,
-            **kwargs,
-        )
+    if rolling_horizon:
+        kwargs["horizon"] = cf_solving.get("horizon", 365)
+        kwargs["overlap"] = cf_solving.get("overlap", 0)
+        n.optimize.optimize_with_rolling_horizon(**kwargs)
+        status, condition = "", ""
+    elif skip_iterations:
+        status, condition = n.optimize(**kwargs)
     else:
+        kwargs["track_iterations"] = (cf_solving.get("track_iterations", False),)
+        kwargs["min_iterations"] = (cf_solving.get("min_iterations", 4),)
+        kwargs["max_iterations"] = (cf_solving.get("max_iterations", 6),)
         status, condition = n.optimize.optimize_transmission_expansion_iteratively(
-            solver_name=solver_name,
-            track_iterations=track_iterations,
-            min_iterations=min_iterations,
-            max_iterations=max_iterations,
-            transmission_losses=transmission_losses,
-            extra_functionality=extra_functionality,
-            multi_investment_periods=multi_horizon,
-            **solver_options,
-            **kwargs,
+            **kwargs
         )
 
-    if status != "ok":
+    if status != "ok" and not rolling_horizon:
         logger.warning(
             f"Solving status '{status}' with termination condition '{condition}'"
         )
@@ -677,11 +672,7 @@ if __name__ == "__main__":
 
     np.random.seed(solve_opts.get("seed", 123))
 
-    if "overrides" in snakemake.input.keys():
-        overrides = override_component_attrs(snakemake.input.overrides)
-        n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
-    else:
-        n = pypsa.Network(snakemake.input.network)
+    n = pypsa.Network(snakemake.input.network)
 
     n = prepare_network(
         n,

scripts/solve_operations_network.py

@@ -11,11 +11,7 @@ import logging
 
 import numpy as np
 import pypsa
-from _helpers import (
-    configure_logging,
-    override_component_attrs,
-    update_config_with_sector_opts,
-)
+from _helpers import configure_logging, update_config_with_sector_opts
 from solve_network import prepare_network, solve_network
 
 logger = logging.getLogger(__name__)
@@ -45,11 +41,7 @@ if __name__ == "__main__":
 
     np.random.seed(solve_opts.get("seed", 123))
 
-    if "overrides" in snakemake.input:
-        overrides = override_component_attrs(snakemake.input.overrides)
-        n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
-    else:
-        n = pypsa.Network(snakemake.input.network)
+    n = pypsa.Network(snakemake.input.network)
 
     n.optimize.fix_optimal_capacities()
     n = prepare_network(n, solve_opts, config=snakemake.config)