From d33547efbab3940b7fafd9cf81b50790a59c1506 Mon Sep 17 00:00:00 2001
From: virio-andreyana <arsyan.virio@gmail.com>
Date: Wed, 8 Feb 2023 11:45:18 +0100
Subject: [PATCH 001/176] importing rioxarray instead of xarray to open
 rasterio file

---
 envs/environment.fixed.yaml  | 1 +
 envs/environment.yaml        | 1 +
 scripts/build_ship_raster.py | 4 ++--
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/envs/environment.fixed.yaml b/envs/environment.fixed.yaml
index 1c7aeaad..94f2c9ce 100644
--- a/envs/environment.fixed.yaml
+++ b/envs/environment.fixed.yaml
@@ -335,6 +335,7 @@ dependencies:
 - requests=2.28.1
 - retry=0.9.2
 - rich=12.5.1
+- rioxarray=0.13.3
 - rtree=1.0.0
 - s2n=1.0.10
 - scikit-learn=1.1.1
diff --git a/envs/environment.yaml b/envs/environment.yaml
index 490268e8..74a63eb8 100644
--- a/envs/environment.yaml
+++ b/envs/environment.yaml
@@ -29,6 +29,7 @@ dependencies:
 - pandas
 - geopandas>=0.11.0
 - xarray
+- rioxarray
 - netcdf4
 - networkx
 - scipy
diff --git a/scripts/build_ship_raster.py b/scripts/build_ship_raster.py
index 0ae134fd..fcfa632d 100644
--- a/scripts/build_ship_raster.py
+++ b/scripts/build_ship_raster.py
@@ -41,7 +41,7 @@ import logging
 import os
 import zipfile
 
-import xarray as xr
+import rioxarray as rio
 from _helpers import configure_logging
 from build_natura_raster import determine_cutout_xXyY
 
@@ -59,7 +59,7 @@ if __name__ == "__main__":
 
     with zipfile.ZipFile(snakemake.input.ship_density) as zip_f:
         zip_f.extract("shipdensity_global.tif")
-        with xr.open_rasterio("shipdensity_global.tif") as ship_density:
+        with rio.open_rasterio("shipdensity_global.tif") as ship_density:
             ship_density = ship_density.drop(["band"]).sel(
                 x=slice(min(xs), max(Xs)), y=slice(max(Ys), min(ys))
             )

From a3df2c12aac00946fddc5fd14da89cb630689b34 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 3 Apr 2023 23:12:09 +0000
Subject: [PATCH 002/176] [pre-commit.ci] pre-commit autoupdate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

updates:
- [github.com/psf/black: 23.1.0 → 23.3.0](https://github.com/psf/black/compare/23.1.0...23.3.0)
---
 .pre-commit-config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 337d047b..e66bcc53 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -51,7 +51,7 @@ repos:
 
   # Formatting with "black" coding style
 - repo: https://github.com/psf/black
-  rev: 23.1.0
+  rev: 23.3.0
   hooks:
       # Format Python files
   - id: black

From da844ddd50a9ab54c29d7f1bb8e43996a0fc2292 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 10 Apr 2023 22:01:58 +0000
Subject: [PATCH 003/176] [pre-commit.ci] pre-commit autoupdate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

updates:
- [github.com/PyCQA/docformatter: v1.5.1 → v1.6.0](https://github.com/PyCQA/docformatter/compare/v1.5.1...v1.6.0)
- [github.com/snakemake/snakefmt: v0.8.3 → v0.8.4](https://github.com/snakemake/snakefmt/compare/v0.8.3...v0.8.4)
---
 .pre-commit-config.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index e66bcc53..7e2f1149 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -39,7 +39,7 @@ repos:
 
   # Make docstrings PEP 257 compliant
 - repo: https://github.com/PyCQA/docformatter
-  rev: v1.5.1
+  rev: v1.6.0
   hooks:
   - id: docformatter
     args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"]
@@ -74,7 +74,7 @@ repos:
 
   # Format Snakemake rule / workflow files
 - repo: https://github.com/snakemake/snakefmt
-  rev: v0.8.3
+  rev: v0.8.4
   hooks:
   - id: snakefmt
 

From 460bbd080f5fd119abacd2cf64c2946dcfd4231c Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 10 Apr 2023 22:02:32 +0000
Subject: [PATCH 004/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/_helpers.py                                |  2 --
 scripts/add_existing_baseyear.py                   |  2 --
 scripts/build_biomass_potentials.py                |  4 ----
 scripts/build_cop_profiles.py                      |  5 +++--
 scripts/build_energy_totals.py                     |  2 --
 scripts/build_gas_network.py                       | 13 +++++++------
 scripts/build_industrial_distribution_key.py       |  3 ---
 scripts/build_industrial_production_per_country.py |  1 -
 scripts/build_retro_cost.py                        |  2 --
 scripts/build_salt_cavern_potentials.py            |  5 ++---
 scripts/build_transport_demand.py                  |  1 -
 scripts/cluster_gas_network.py                     |  1 -
 scripts/make_summary.py                            |  2 --
 scripts/plot_summary.py                            |  1 -
 scripts/prepare_sector_network.py                  |  4 ----
 scripts/simplify_network.py                        |  9 +++------
 16 files changed, 15 insertions(+), 42 deletions(-)

diff --git a/scripts/_helpers.py b/scripts/_helpers.py
index 281ee9c2..dc10eac1 100644
--- a/scripts/_helpers.py
+++ b/scripts/_helpers.py
@@ -400,7 +400,6 @@ def override_component_attrs(directory):
     -------
     Dictionary of overridden component attributes.
     """
-
     attrs = Dict({k: v.copy() for k, v in component_attrs.items()})
 
     for component, list_name in components.list_name.items():
@@ -418,7 +417,6 @@ def generate_periodic_profiles(dt_index, nodes, weekly_profile, localize=None):
     country for the period dt_index, taking account of time zones and summer
     time.
     """
-
     weekly_profile = pd.Series(weekly_profile, range(24 * 7))
 
     week_df = pd.DataFrame(index=dt_index, columns=nodes)
diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py
index a81d606d..992f3f98 100644
--- a/scripts/add_existing_baseyear.py
+++ b/scripts/add_existing_baseyear.py
@@ -38,7 +38,6 @@ def add_build_year_to_new_assets(n, baseyear):
     baseyear : int
         year in which optimized assets are built
     """
-
     # Give assets with lifetimes and no build year the build year baseyear
     for c in n.iterate_components(["Link", "Generator", "Store"]):
         assets = c.df.index[(c.df.lifetime != np.inf) & (c.df.build_year == 0)]
@@ -62,7 +61,6 @@ def add_existing_renewables(df_agg):
     Append existing renewables to the df_agg pd.DataFrame with the conventional
     power plants.
     """
-
     carriers = {"solar": "solar", "onwind": "onwind", "offwind": "offwind-ac"}
 
     for tech in ["solar", "onwind", "offwind"]:
diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py
index 7387418a..50f2583c 100644
--- a/scripts/build_biomass_potentials.py
+++ b/scripts/build_biomass_potentials.py
@@ -68,7 +68,6 @@ def enspreso_biomass_potentials(year=2020, scenario="ENS_Low"):
         Biomass potentials for given year and scenario
         in TWh/a by commodity and NUTS2 region.
     """
-
     glossary = pd.read_excel(
         str(snakemake.input.enspreso_biomass),
         sheet_name="Glossary",
@@ -124,7 +123,6 @@ def disaggregate_nuts0(bio):
     -------
     pd.DataFrame
     """
-
     pop = build_nuts_population_data()
 
     # get population in nuts2
@@ -149,7 +147,6 @@ def build_nuts2_shapes():
     - add RS, AL, BA country shapes (not covered in NUTS 2013)
     - consistently name ME, MK
     """
-
     nuts2 = gpd.GeoDataFrame(
         gpd.read_file(snakemake.input.nuts2).set_index("id").geometry
     )
@@ -186,7 +183,6 @@ def convert_nuts2_to_regions(bio_nuts2, regions):
     -------
     gpd.GeoDataFrame
     """
-
     # calculate area of nuts2 regions
     bio_nuts2["area_nuts2"] = area(bio_nuts2)
 
diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py
index fa04e4a1..94cc8fb4 100644
--- a/scripts/build_cop_profiles.py
+++ b/scripts/build_cop_profiles.py
@@ -7,10 +7,11 @@
 Build coefficient of performance (COP) time series for air- or ground-sourced
 heat pumps.
 
-The COP is a function of the temperature difference between
-source and sink.
+The COP is a function of the temperature difference between source and
+sink.
 
 The quadratic regression used is based on Staffell et al. (2012)
+
 https://doi.org/10.1039/C2EE22653G.
 """
 
diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py
index 1480732a..f11c134f 100644
--- a/scripts/build_energy_totals.py
+++ b/scripts/build_energy_totals.py
@@ -124,7 +124,6 @@ def build_eurostat(input_eurostat, countries, report_year, year):
     """
     Return multi-index for all countries' energy data in TWh/a.
     """
-
     filenames = {
         2016: f"/{year}-Energy-Balances-June2016edition.xlsx",
         2017: f"/{year}-ENERGY-BALANCES-June2017edition.xlsx",
@@ -163,7 +162,6 @@ def build_swiss(year):
     """
     Return a pd.Series of Swiss energy data in TWh/a.
     """
-
     fn = snakemake.input.swiss
 
     df = pd.read_csv(fn, index_col=[0, 1]).loc["CH", str(year)]
diff --git a/scripts/build_gas_network.py b/scripts/build_gas_network.py
index b34ebe9f..907b151c 100644
--- a/scripts/build_gas_network.py
+++ b/scripts/build_gas_network.py
@@ -22,14 +22,15 @@ def diameter_to_capacity(pipe_diameter_mm):
     """
     Calculate pipe capacity in MW based on diameter in mm.
 
-    20 inch (500 mm)  50 bar -> 1.5   GW CH4 pipe capacity (LHV)
-    24 inch (600 mm)  50 bar -> 5     GW CH4 pipe capacity (LHV)
-    36 inch (900 mm)  50 bar -> 11.25 GW CH4 pipe capacity (LHV)
-    48 inch (1200 mm) 80 bar -> 21.7  GW CH4 pipe capacity (LHV)
+    20 inch (500 mm)  50 bar -> 1.5   GW CH4 pipe capacity (LHV) 24 inch
+    (600 mm)  50 bar -> 5     GW CH4 pipe capacity (LHV) 36 inch (900
+    mm)  50 bar -> 11.25 GW CH4 pipe capacity (LHV) 48 inch (1200 mm) 80
+    bar -> 21.7  GW CH4 pipe capacity (LHV)
 
-    Based on p.15 of https://gasforclimate2050.eu/wp-content/uploads/2020/07/2020_European-Hydrogen-Backbone_Report.pdf
+    Based on p.15 of
+
+    https://gasforclimate2050.eu/wp-content/uploads/2020/07/2020_European-Hydrogen-Backbone_Report.pdf
     """
-
     # slopes definitions
     m0 = (1500 - 0) / (500 - 0)
     m1 = (5000 - 1500) / (600 - 500)
diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py
index 5f2f6893..d4518d1c 100644
--- a/scripts/build_industrial_distribution_key.py
+++ b/scripts/build_industrial_distribution_key.py
@@ -27,7 +27,6 @@ def locate_missing_industrial_sites(df):
     Should only be used if the model's spatial resolution is coarser
     than individual cities.
     """
-
     try:
         from geopy.extra.rate_limiter import RateLimiter
         from geopy.geocoders import Nominatim
@@ -71,7 +70,6 @@ def prepare_hotmaps_database(regions):
     """
     Load hotmaps database of industrial sites and map onto bus regions.
     """
-
     df = pd.read_csv(snakemake.input.hotmaps_industrial_database, sep=";", index_col=0)
 
     df[["srid", "coordinates"]] = df.geom.str.split(";", expand=True)
@@ -103,7 +101,6 @@ def build_nodal_distribution_key(hotmaps, regions, countries):
     """
     Build nodal distribution keys for each sector.
     """
-
     sectors = hotmaps.Subsector.unique()
 
     keys = pd.DataFrame(index=regions.index, columns=sectors, dtype=float)
diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py
index 8a4c19c2..1c2e4557 100644
--- a/scripts/build_industrial_production_per_country.py
+++ b/scripts/build_industrial_production_per_country.py
@@ -247,7 +247,6 @@ def separate_basic_chemicals(demand, year):
     """
     Separate basic chemicals into ammonia, chlorine, methanol and HVC.
     """
-
     ammonia = pd.read_csv(snakemake.input.ammonia_production, index_col=0)
 
     there = ammonia.index.intersection(demand.index)
diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py
index 037139d1..72792c92 100644
--- a/scripts/build_retro_cost.py
+++ b/scripts/build_retro_cost.py
@@ -150,7 +150,6 @@ def prepare_building_stock_data():
                        type and period
 
     """
-
     building_data = pd.read_csv(snakemake.input.building_stock, usecols=list(range(13)))
 
     # standardize data
@@ -318,7 +317,6 @@ def prepare_building_topology(u_values, same_building_topology=True):
     Reads in typical building topologies (e.g. average surface of building
     elements) and typical losses through thermal bridging and air ventilation.
     """
-
     data_tabula = pd.read_csv(
         snakemake.input.data_tabula,
         skiprows=lambda x: x in range(1, 11),
diff --git a/scripts/build_salt_cavern_potentials.py b/scripts/build_salt_cavern_potentials.py
index 082c688f..3fed7f91 100644
--- a/scripts/build_salt_cavern_potentials.py
+++ b/scripts/build_salt_cavern_potentials.py
@@ -6,8 +6,8 @@
 """
 Build salt cavern potentials for hydrogen storage.
 
-Technical Potential of Salt Caverns for Hydrogen Storage in Europe
-CC-BY 4.0
+Technical Potential of Salt Caverns for Hydrogen Storage in Europe CC-BY
+4.0
 https://doi.org/10.20944/preprints201910.0187.v1
 https://doi.org/10.1016/j.ijhydene.2019.12.161
 
@@ -39,7 +39,6 @@ def load_bus_regions(onshore_path, offshore_path):
     """
     Load pypsa-eur on- and offshore regions and concat.
     """
-
     bus_regions_offshore = gpd.read_file(offshore_path)
     bus_regions_onshore = gpd.read_file(onshore_path)
     bus_regions = concat_gdf([bus_regions_offshore, bus_regions_onshore])
diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py
index a684036d..29973da3 100644
--- a/scripts/build_transport_demand.py
+++ b/scripts/build_transport_demand.py
@@ -124,7 +124,6 @@ def bev_availability_profile(fn, snapshots, nodes, options):
     """
     Derive plugged-in availability for passenger electric vehicles.
     """
-
     traffic = pd.read_csv(fn, skiprows=2, usecols=["count"]).squeeze("columns")
 
     avail_max = options["bev_avail_max"]
diff --git a/scripts/cluster_gas_network.py b/scripts/cluster_gas_network.py
index d180041c..44188638 100755
--- a/scripts/cluster_gas_network.py
+++ b/scripts/cluster_gas_network.py
@@ -30,7 +30,6 @@ def load_bus_regions(onshore_path, offshore_path):
     """
     Load pypsa-eur on- and offshore regions and concat.
     """
-
     bus_regions_offshore = gpd.read_file(offshore_path)
     bus_regions_onshore = gpd.read_file(onshore_path)
     bus_regions = concat_gdf([bus_regions_offshore, bus_regions_onshore])
diff --git a/scripts/make_summary.py b/scripts/make_summary.py
index 5d1ee3dc..7b33aed8 100644
--- a/scripts/make_summary.py
+++ b/scripts/make_summary.py
@@ -320,7 +320,6 @@ def calculate_supply(n, label, supply):
     Calculate the max dispatch of each component at the buses aggregated by
     carrier.
     """
-
     bus_carriers = n.buses.carrier.unique()
 
     for i in bus_carriers:
@@ -372,7 +371,6 @@ def calculate_supply_energy(n, label, supply_energy):
     Calculate the total energy supply/consuption of each component at the buses
     aggregated by carrier.
     """
-
     bus_carriers = n.buses.carrier.unique()
 
     for i in bus_carriers:
diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py
index af028116..339a8457 100644
--- a/scripts/plot_summary.py
+++ b/scripts/plot_summary.py
@@ -441,7 +441,6 @@ def plot_carbon_budget_distribution(input_eurostat):
     """
     Plot historical carbon emissions in the EU and decarbonization path.
     """
-
     import seaborn as sns
 
     sns.set()
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index da6eab72..5937eb96 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -51,7 +51,6 @@ def define_spatial(nodes, options):
     ----------
     nodes : list-like
     """
-
     global spatial
 
     spatial.nodes = nodes
@@ -362,7 +361,6 @@ def update_wind_solar_costs(n, costs):
     Update costs for wind and solar generators added with pypsa-eur to those
     cost in the planning year.
     """
-
     # NB: solar costs are also manipulated for rooftop
     # when distribution grid is inserted
     n.generators.loc[n.generators.carrier == "solar", "capital_cost"] = costs.at[
@@ -440,7 +438,6 @@ def add_carrier_buses(n, carrier, nodes=None):
     """
     Add buses to connect e.g. coal, nuclear and oil plants.
     """
-
     if nodes is None:
         nodes = vars(spatial)[carrier].nodes
     location = vars(spatial)[carrier].locations
@@ -487,7 +484,6 @@ def remove_elec_base_techs(n):
     batteries and H2) from base electricity-only network, since they're added
     here differently using links.
     """
-
     for c in n.iterate_components(snakemake.config["pypsa_eur"]):
         to_keep = snakemake.config["pypsa_eur"][c.name]
         to_remove = pd.Index(c.df.carrier.unique()).symmetric_difference(to_keep)
diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 5e50c4ab..58c4ad17 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -112,15 +112,12 @@ def simplify_network_to_380(n):
     Fix all lines to a voltage level of 380 kV and remove all transformers.
 
     The function preserves the transmission capacity for each line while
-    updating
-    its voltage level, line type and number of parallel bundles
+    updating its voltage level, line type and number of parallel bundles
     (num_parallel).
 
     Transformers are removed and connected components are moved from
-    their
-    starting bus to their ending bus. The corresponding starting buses
-    are
-    removed as well.
+    their starting bus to their ending bus. The corresponding starting
+    buses are removed as well.
     """
     logger.info("Mapping all network lines onto a single 380kV layer")
 

From 9d61e9680da38c734b91507e1e773d6a97a93008 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Tue, 11 Apr 2023 02:03:54 +0200
Subject: [PATCH 005/176] Move all config files to configs folder

Note: this also includes the test config files. Update must be made for the tutorial
---
 configs/config.default.yaml          | 959 +++++++++++++++++++++++++++
 configs/test/config.electricity.yaml |  80 +++
 configs/test/config.myopic.yaml      |  79 +++
 configs/test/config.overnight.yaml   |  74 +++
 4 files changed, 1192 insertions(+)
 create mode 100644 configs/config.default.yaml
 create mode 100644 configs/test/config.electricity.yaml
 create mode 100644 configs/test/config.myopic.yaml
 create mode 100644 configs/test/config.overnight.yaml

diff --git a/configs/config.default.yaml b/configs/config.default.yaml
new file mode 100644
index 00000000..5218bc22
--- /dev/null
+++ b/configs/config.default.yaml
@@ -0,0 +1,959 @@
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: CC0-1.0
+
+version: 0.8.0
+tutorial: false
+
+logging:
+  level: INFO
+  format: '%(levelname)s:%(name)s:%(message)s'
+
+run:
+  name: "" # use this to keep track of runs with different settings
+  disable_progressbar: false # set to true to disable the progressbar
+  shared_resources: false # set to true to share the default resources across runs
+  shared_cutouts: true # set to true to share the default cutout(s) across runs
+
+foresight: overnight # options are overnight, myopic, perfect (perfect is not yet implemented)
+# if you use myopic or perfect foresight, set the investment years in "planning_horizons" below
+
+scenario:
+  simpl:
+  - ''
+  ll:  # allowed transmission line volume expansion, can be any float >= 1.0 with a prefix v|c (today) or "copt"
+  - v1.0
+  - v1.5
+  clusters:  # number of nodes in Europe, any integer between 37 (1 node per country-zone) and several hundred
+  - 37
+  - 128
+  - 256
+  - 512
+  - 1024
+  opts: # only relevant for PyPSA-Eur
+  - ''
+  sector_opts: # this is where the main scenario settings are
+  - Co2L0-3H-T-H-B-I-A-solar+p3-dist1
+  # to really understand the options here, look in scripts/prepare_sector_network.py
+  # Co2Lx specifies the CO2 target in x% of the 1990 values; default will give default (5%);
+  # Co2L0p25 will give 25% CO2 emissions; Co2Lm0p05 will give 5% negative emissions
+  # xH is the temporal resolution; 3H is 3-hourly, i.e. one snapshot every 3 hours
+  # single letters are sectors: T for land transport, H for building heating,
+  # B for biomass supply, I for industry, shipping and aviation,
+  # A for agriculture, forestry and fishing
+  # solar+c0.5 reduces the capital cost of solar to 50\% of reference value
+  # solar+p3 multiplies the available installable potential by factor 3
+  # seq400 sets the potential of CO2 sequestration to 400 Mt CO2 per year
+  # dist{n} includes distribution grids with investment cost of n times cost in data/costs.csv
+  # for myopic/perfect foresight cb states the carbon budget in GtCO2 (cumulative
+  # emissions throughout the transition path in the timeframe determined by the
+  # planning_horizons), be:beta decay; ex:exponential decay
+  # cb40ex0 distributes a carbon budget of 40 GtCO2 following an exponential
+  # decay with initial growth rate 0
+  planning_horizons: # investment years for myopic and perfect; for overnight, year of cost assumptions can be different and is defined under 'costs'
+  - 2050
+  # for example, set to
+  # - 2020
+  # - 2030
+  # - 2040
+  # - 2050
+  # for myopic foresight
+
+countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK']
+
+snapshots:
+  start: "2013-01-01"
+  end: "2014-01-01"
+  inclusive: 'left' # include start, not end
+
+enable:
+  prepare_links_p_nom: false
+  retrieve_databundle: true
+  retrieve_sector_databundle: true
+  retrieve_cost_data: true
+  build_cutout: false
+  retrieve_cutout: true
+  build_natura_raster: false
+  retrieve_natura_raster: true
+  custom_busmap: false
+
+# CO2 budget as a fraction of 1990 emissions
+# this is over-ridden if CO2Lx is set in sector_opts
+# this is also over-ridden if cb is set in sector_opts
+co2_budget:
+  2020: 0.701
+  2025: 0.524
+  2030: 0.297
+  2035: 0.150
+  2040: 0.071
+  2045: 0.032
+  2050: 0.000
+
+electricity:
+  voltages: [220., 300., 380.]
+  gaslimit: false # global gas usage limit of X MWh_th
+  co2limit: 7.75e+7 # 0.05 * 3.1e9*0.5
+  co2base: 1.487e+9
+  agg_p_nom_limits: data/agg_p_nom_minmax.csv
+
+  operational_reserve: # like https://genxproject.github.io/GenX/dev/core/#Reserves
+    activate: false
+    epsilon_load: 0.02 # share of total load
+    epsilon_vres: 0.02 # share of total renewable supply
+    contingency: 4000 # fixed capacity in MW
+
+  max_hours:
+    battery: 6
+    H2: 168
+
+  extendable_carriers:
+    Generator: [solar, onwind, offwind-ac, offwind-dc, OCGT]
+    StorageUnit: [] # battery, H2
+    Store: [battery, H2]
+    Link: [] # H2 pipeline
+
+  # use pandas query strings here, e.g. Country not in ['Germany']
+  powerplants_filter: (DateOut >= 2022 or DateOut != DateOut)
+  # use pandas query strings here, e.g. Country in ['Germany']
+  custom_powerplants: false
+
+  conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass]
+  renewable_carriers: [solar, onwind, offwind-ac, offwind-dc, hydro]
+
+  estimate_renewable_capacities:
+    enable: true
+    # Add capacities from OPSD data
+    from_opsd: true
+    # Renewable capacities are based on existing capacities reported by IRENA
+    year: 2020
+    # Artificially limit maximum capacities to factor * (IRENA capacities),
+    # i.e. 110% of <years>'s capacities => expansion_limit: 1.1
+    # false: Use estimated renewable potentials determine by the workflow
+    expansion_limit: false
+    technology_mapping:
+      # Wind is the Fueltype in powerplantmatching, onwind, offwind-{ac,dc} the carrier in PyPSA-Eur
+      Offshore: [offwind-ac, offwind-dc]
+      Onshore: [onwind]
+      PV: [solar]
+
+
+atlite:
+  default_cutout: europe-2013-era5
+  nprocesses: 4
+  show_progress: false # false saves time
+  cutouts:
+    # use 'base' to determine geographical bounds and time span from config
+    # base:
+      # module: era5
+    europe-2013-era5:
+      module: era5 # in priority order
+      x: [-12., 35.]
+      y: [33., 72]
+      dx: 0.3
+      dy: 0.3
+      time: ['2013', '2013']
+    europe-2013-sarah:
+      module: [sarah, era5] # in priority order
+      x: [-12., 45.]
+      y: [33., 65]
+      dx: 0.2
+      dy: 0.2
+      time: ['2013', '2013']
+      sarah_interpolate: false
+      sarah_dir:
+      features: [influx, temperature]
+
+
+renewable:
+  onwind:
+    cutout: europe-2013-era5
+    resource:
+      method: wind
+      turbine: Vestas_V112_3MW
+    capacity_per_sqkm: 3 # ScholzPhd Tab 4.3.1: 10MW/km^2 and assuming 30% fraction of the already restricted
+    # area is available for installation of wind generators due to competing land use and likely public
+    # acceptance issues.
+    # correction_factor: 0.93
+    corine:
+      # Scholz, Y. (2012). Renewable energy based electricity supply at low costs
+      #  development of the REMix model and application for Europe. ( p.42 / p.28)
+      grid_codes: [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32]
+      distance: 1000
+      distance_grid_codes: [1, 2, 3, 4, 5, 6]
+    natura: true
+    excluder_resolution: 100
+    potential: simple # or conservative
+    clip_p_max_pu: 1.e-2
+  offwind-ac:
+    cutout: europe-2013-era5
+    resource:
+      method: wind
+      turbine: NREL_ReferenceTurbine_5MW_offshore
+    capacity_per_sqkm: 2 # ScholzPhd Tab 4.3.1: 10MW/km^2 and assuming 20% fraction of the already restricted
+    # area is available for installation of wind generators due to competing land use and likely public
+    # acceptance issues.
+    correction_factor: 0.8855
+    # proxy for wake losses
+    # from 10.1016/j.energy.2018.08.153
+    # until done more rigorously in #153
+    corine: [44, 255]
+    natura: true
+    ship_threshold: 400
+    max_depth: 50
+    max_shore_distance: 30000
+    excluder_resolution: 200
+    potential: simple # or conservative
+    clip_p_max_pu: 1.e-2
+  offwind-dc:
+    cutout: europe-2013-era5
+    resource:
+      method: wind
+      turbine: NREL_ReferenceTurbine_5MW_offshore
+    capacity_per_sqkm: 2 # ScholzPhd Tab 4.3.1: 10MW/km^2 and assuming 20% fraction of the already restricted
+    # area is available for installation of wind generators due to competing land use and likely public
+    # acceptance issues.
+    correction_factor: 0.8855
+    # proxy for wake losses
+    # from 10.1016/j.energy.2018.08.153
+    # until done more rigorously in #153
+    corine: [44, 255]
+    natura: true
+    ship_threshold: 400
+    max_depth: 50
+    min_shore_distance: 30000
+    excluder_resolution: 200
+    potential: simple # or conservative
+    clip_p_max_pu: 1.e-2
+  solar:
+    cutout: europe-2013-sarah
+    resource:
+      method: pv
+      panel: CSi
+      orientation:
+        slope: 35.
+        azimuth: 180.
+    capacity_per_sqkm: 1.7 # ScholzPhd Tab 4.3.1: 170 MW/km^2 and assuming 1% of the area can be used for solar PV panels
+    # Correction factor determined by comparing uncorrected area-weighted full-load hours to those
+    # published in Supplementary Data to
+    # Pietzcker, Robert Carl, et al. "Using the sun to decarbonize the power
+    # sector -- The economic potential of photovoltaics and concentrating solar
+    # power." Applied Energy 135 (2014): 704-720.
+    # This correction factor of 0.854337 may be in order if using reanalysis data.
+    # for discussion refer to https://github.com/PyPSA/pypsa-eur/pull/304
+    # correction_factor: 0.854337
+    corine: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 26, 31, 32]
+    natura: true
+    excluder_resolution: 100
+    potential: simple # or conservative
+    clip_p_max_pu: 1.e-2
+  hydro:
+    cutout: europe-2013-era5
+    carriers: [ror, PHS, hydro]
+    PHS_max_hours: 6
+    hydro_max_hours: "energy_capacity_totals_by_country" # one of energy_capacity_totals_by_country, estimate_by_large_installations or a float
+    clip_min_inflow: 1.0
+
+conventional:
+  nuclear:
+    p_max_pu: "data/nuclear_p_max_pu.csv" # float of file name
+
+lines:
+  types:
+    220.: "Al/St 240/40 2-bundle 220.0"
+    300.: "Al/St 240/40 3-bundle 300.0"
+    380.: "Al/St 240/40 4-bundle 380.0"
+  s_max_pu: 0.7
+  s_nom_max: .inf
+  length_factor: 1.25
+  under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity
+
+links:
+  p_max_pu: 1.0
+  p_nom_max: .inf
+  include_tyndp: true
+  under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity
+
+transformers:
+  x: 0.1
+  s_nom: 2000.
+  type: ''
+
+load:
+  power_statistics: true # only for files from <2019; set false in order to get ENTSOE transparency data
+  interpolate_limit: 3 # data gaps up until this size are interpolated linearly
+  time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from
+  manual_adjustments: true # false
+  scaling_factor: 1.0
+
+# regulate what components with which carriers are kept from PyPSA-Eur;
+# some technologies are removed because they are implemented differently
+# (e.g. battery or H2 storage) or have different year-dependent costs
+# in PyPSA-Eur-Sec
+pypsa_eur:
+  Bus:
+  - AC
+  Link:
+  - DC
+  Generator:
+  - onwind
+  - offwind-ac
+  - offwind-dc
+  - solar
+  - ror
+  StorageUnit:
+  - PHS
+  - hydro
+  Store: []
+
+energy:
+  energy_totals_year: 2011
+  base_emissions_year: 1990
+  eurostat_report_year: 2016
+  emissions: CO2 # "CO2" or "All greenhouse gases - (CO2 equivalent)"
+
+biomass:
+  year: 2030
+  scenario: ENS_Med
+  classes:
+    solid biomass:
+    - Agricultural waste
+    - Fuelwood residues
+    - Secondary Forestry residues - woodchips
+    - Sawdust
+    - Residues from landscape care
+    - Municipal waste
+    not included:
+    - Sugar from sugar beet
+    - Rape seed
+    - "Sunflower, soya seed "
+    - Bioethanol barley, wheat, grain maize, oats, other cereals and rye
+    - Miscanthus, switchgrass, RCG
+    - Willow
+    - Poplar
+    - FuelwoodRW
+    - C&P_RW
+    biogas:
+    - Manure solid, liquid
+    - Sludge
+
+
+solar_thermal:
+  clearsky_model: simple  # should be "simple" or "enhanced"?
+  orientation:
+    slope: 45.
+    azimuth: 180.
+
+# only relevant for foresight = myopic or perfect
+existing_capacities:
+  grouping_years_power: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030]
+  grouping_years_heat: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019] # these should not extend 2020
+  threshold_capacity: 10
+  conventional_carriers:
+  - lignite
+  - coal
+  - oil
+  - uranium
+
+
+sector:
+  district_heating:
+    potential: 0.6  # maximum fraction of urban demand which can be supplied by district heating
+     # increase of today's district heating demand to potential maximum district heating share
+     # progress = 0 means today's district heating share, progress = 1 means maximum fraction of urban demand is supplied by district heating
+    progress:
+      2020: 0.0
+      2030: 0.3
+      2040: 0.6
+      2050: 1.0
+    district_heating_loss: 0.15
+  cluster_heat_buses: false # cluster residential and service heat buses to one to save memory
+  bev_dsm_restriction_value: 0.75  #Set to 0 for no restriction on BEV DSM
+  bev_dsm_restriction_time: 7  #Time at which SOC of BEV has to be dsm_restriction_value
+  transport_heating_deadband_upper: 20.
+  transport_heating_deadband_lower: 15.
+  ICE_lower_degree_factor: 0.375  #in per cent increase in fuel consumption per degree above deadband
+  ICE_upper_degree_factor: 1.6
+  EV_lower_degree_factor: 0.98
+  EV_upper_degree_factor: 0.63
+  bev_dsm: true #turns on EV battery
+  bev_availability: 0.5  #How many cars do smart charging
+  bev_energy: 0.05  #average battery size in MWh
+  bev_charge_efficiency: 0.9  #BEV (dis-)charging efficiency
+  bev_plug_to_wheel_efficiency: 0.2 #kWh/km from EPA https://www.fueleconomy.gov/feg/ for Tesla Model S
+  bev_charge_rate: 0.011 #3-phase charger with 11 kW
+  bev_avail_max: 0.95
+  bev_avail_mean: 0.8
+  v2g: true #allows feed-in to grid from EV battery
+  #what is not EV or FCEV is oil-fuelled ICE
+  land_transport_fuel_cell_share:
+    2020: 0
+    2030: 0.05
+    2040: 0.1
+    2050: 0.15
+  land_transport_electric_share:
+    2020: 0
+    2030: 0.25
+    2040: 0.6
+    2050: 0.85
+  land_transport_ice_share:
+    2020: 1
+    2030: 0.7
+    2040: 0.3
+    2050: 0
+  transport_fuel_cell_efficiency: 0.5
+  transport_internal_combustion_efficiency: 0.3
+  agriculture_machinery_electric_share: 0
+  agriculture_machinery_oil_share: 1
+  agriculture_machinery_fuel_efficiency: 0.7 # fuel oil per use
+  agriculture_machinery_electric_efficiency: 0.3 # electricity per use
+  MWh_MeOH_per_MWh_H2: 0.8787 # in LHV, source: DECHEMA (2017): Low carbon energy and feedstock for the European chemical industry , pg. 64.
+  MWh_MeOH_per_tCO2: 4.0321 # in LHV, source: DECHEMA (2017): Low carbon energy and feedstock for the European chemical industry , pg. 64.
+  MWh_MeOH_per_MWh_e: 3.6907 # in LHV, source: DECHEMA (2017): Low carbon energy and feedstock for the European chemical industry , pg. 64.
+  shipping_hydrogen_liquefaction: false # whether to consider liquefaction costs for shipping H2 demands
+  shipping_hydrogen_share:
+    2020: 0
+    2030: 0
+    2040: 0
+    2050: 0
+  shipping_methanol_share:
+    2020: 0
+    2030: 0.3
+    2040: 0.7
+    2050: 1
+  shipping_oil_share:
+    2020: 1
+    2030: 0.7
+    2040: 0.3
+    2050: 0
+  shipping_methanol_efficiency: 0.46 # 10-15% higher https://www.iea-amf.org/app/webroot/files/file/Annex%20Reports/AMF_Annex_56.pdf, https://users.ugent.be/~lsileghe/documents/extended_abstract.pdf
+  shipping_oil_efficiency: 0.40 #For conversion of fuel oil to propulsion in 2011
+  aviation_demand_factor: 1. # relative aviation demand compared to today
+  HVC_demand_factor: 1. # relative HVC demand compared to today
+  time_dep_hp_cop: true #time dependent heat pump coefficient of performance
+  heat_pump_sink_T: 55. # Celsius, based on DTU / large area radiators; used in build_cop_profiles.py
+   # conservatively high to cover hot water and space heating in poorly-insulated buildings
+  reduce_space_heat_exogenously: true  # reduces space heat demand by a given factor (applied before losses in DH)
+  # this can represent e.g. building renovation, building demolition, or if
+  # the factor is negative: increasing floor area, increased thermal comfort, population growth
+  reduce_space_heat_exogenously_factor:  # per unit reduction in space heat demand
+  # the default factors are determined by the LTS scenario from http://tool.european-calculator.eu/app/buildings/building-types-area/?levers=1ddd4444421213bdbbbddd44444ffffff11f411111221111211l212221
+    2020: 0.10  # this results in a space heat demand reduction of 10%
+    2025: 0.09  # first heat demand increases compared to 2020 because of larger floor area per capita
+    2030: 0.09
+    2035: 0.11
+    2040: 0.16
+    2045: 0.21
+    2050: 0.29
+  retrofitting:   # co-optimises building renovation to reduce space heat demand
+    retro_endogen: false  # co-optimise space heat savings
+    cost_factor: 1.0   # weight costs for building renovation
+    interest_rate: 0.04  # for investment in building components
+    annualise_cost: true  # annualise the investment costs
+    tax_weighting: false   # weight costs depending on taxes in countries
+    construction_index: true   # weight costs depending on labour/material costs per country
+  tes: true
+  tes_tau: # 180 day time constant for centralised, 3 day for decentralised
+    decentral: 3
+    central: 180
+  boilers: true
+  oil_boilers: false
+  biomass_boiler: true
+  chp: true
+  micro_chp: false
+  solar_thermal: true
+  solar_cf_correction: 0.788457  # =  >>> 1/1.2683
+  marginal_cost_storage: 0. #1e-4
+  methanation: true
+  helmeth: true
+  coal_cc: false
+  dac: true
+  co2_vent: false
+  allam_cycle: false
+  SMR: true
+  regional_co2_sequestration_potential:
+    enable: false  # enable regionally resolved geological co2 storage potential
+    attribute: 'conservative estimate Mt'
+    include_onshore: false  # include onshore sequestration potentials
+    min_size: 3 # Gt, sites with lower potential will be excluded
+    max_size: 25 # Gt, max sequestration potential for any one site, TODO research suitable value
+    years_of_storage: 25 # years until potential exhausted at optimised annual rate
+  co2_sequestration_potential: 200  #MtCO2/a sequestration potential for Europe
+  co2_sequestration_cost: 10   #EUR/tCO2 for sequestration of CO2
+  co2_spatial: false
+  co2network: false
+  cc_fraction: 0.9  # default fraction of CO2 captured with post-combustion capture
+  hydrogen_underground_storage: true
+  hydrogen_underground_storage_locations:
+    # - onshore  # more than 50 km from sea
+  - nearshore    # within 50 km of sea
+    # - offshore
+  ammonia: false # can be false (no NH3 carrier), true (copperplated NH3), "regional" (regionalised NH3 without network)
+  min_part_load_fischer_tropsch: 0.9 # p_min_pu
+  min_part_load_methanolisation: 0.5 # p_min_pu
+  use_fischer_tropsch_waste_heat: true
+  use_fuel_cell_waste_heat: true
+  use_electrolysis_waste_heat: false
+  electricity_distribution_grid: true
+  electricity_distribution_grid_cost_factor: 1.0  #multiplies cost in data/costs.csv
+  electricity_grid_connection: true  # only applies to onshore wind and utility PV
+  H2_network: true
+  gas_network: false
+  H2_retrofit: false  # if set to True existing gas pipes can be retrofitted to H2 pipes
+  # according to hydrogen backbone strategy (April, 2020) p.15
+  # https://gasforclimate2050.eu/wp-content/uploads/2020/07/2020_European-Hydrogen-Backbone_Report.pdf
+  # 60% of original natural gas capacity could be used in cost-optimal case as H2 capacity
+  H2_retrofit_capacity_per_CH4: 0.6  # ratio for H2 capacity per original CH4 capacity of retrofitted pipelines
+  gas_network_connectivity_upgrade: 1 # https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation
+  gas_distribution_grid: true
+  gas_distribution_grid_cost_factor: 1.0  #multiplies cost in data/costs.csv
+  biomass_spatial: false  # regionally resolve biomass (e.g. potentials)
+  biomass_transport: false  # allow transport of solid biomass between nodes
+  conventional_generation: # generator : carrier
+    OCGT: gas
+  biomass_to_liquid: false
+  biosng: false
+
+industry:
+  St_primary_fraction:  # fraction of steel produced via primary route versus secondary route (scrap+EAF); today fraction is 0.6
+    2020: 0.6
+    2025: 0.55
+    2030: 0.5
+    2035: 0.45
+    2040: 0.4
+    2045: 0.35
+    2050: 0.3
+  DRI_fraction:  # fraction of the primary route converted to DRI + EAF
+    2020: 0
+    2025: 0
+    2030: 0.05
+    2035: 0.2
+    2040: 0.4
+    2045: 0.7
+    2050: 1
+  H2_DRI: 1.7   #H2 consumption in Direct Reduced Iron (DRI),  MWh_H2,LHV/ton_Steel from 51kgH2/tSt in Vogl et al (2018) doi:10.1016/j.jclepro.2018.08.279
+  elec_DRI: 0.322   #electricity consumption in Direct Reduced Iron (DRI) shaft, MWh/tSt HYBRIT brochure https://ssabwebsitecdn.azureedge.net/-/media/hybrit/files/hybrit_brochure.pdf
+  Al_primary_fraction:  # fraction of aluminium produced via the primary route versus scrap; today fraction is 0.4
+    2020: 0.4
+    2025: 0.375
+    2030: 0.35
+    2035: 0.325
+    2040: 0.3
+    2045: 0.25
+    2050: 0.2
+  MWh_NH3_per_tNH3: 5.166 # LHV
+  MWh_CH4_per_tNH3_SMR: 10.8 # 2012's demand from https://ec.europa.eu/docsroom/documents/4165/attachments/1/translations/en/renditions/pdf
+  MWh_elec_per_tNH3_SMR: 0.7 # same source, assuming 94-6% split methane-elec of total energy demand 11.5 MWh/tNH3
+  MWh_H2_per_tNH3_electrolysis: 6.5 # from https://doi.org/10.1016/j.joule.2018.04.017, around 0.197 tH2/tHN3 (>3/17 since some H2 lost and used for energy)
+  MWh_elec_per_tNH3_electrolysis: 1.17 # from https://doi.org/10.1016/j.joule.2018.04.017 Table 13 (air separation and HB)
+  MWh_NH3_per_MWh_H2_cracker: 1.46 # https://github.com/euronion/trace/blob/44a5ff8401762edbef80eff9cfe5a47c8d3c8be4/data/efficiencies.csv
+  NH3_process_emissions: 24.5 # in MtCO2/a from SMR for H2 production for NH3 from UNFCCC for 2015 for EU28
+  petrochemical_process_emissions: 25.5 # in MtCO2/a for petrochemical and other from UNFCCC for 2015 for EU28
+  HVC_primary_fraction: 1. # fraction of today's HVC produced via primary route
+  HVC_mechanical_recycling_fraction: 0. # fraction of today's HVC produced via mechanical recycling
+  HVC_chemical_recycling_fraction: 0. # fraction of today's HVC produced via chemical recycling
+  HVC_production_today: 52. # MtHVC/a from DECHEMA (2017), Figure 16, page 107; includes ethylene, propylene and BTX
+  MWh_elec_per_tHVC_mechanical_recycling: 0.547 # from SI of https://doi.org/10.1016/j.resconrec.2020.105010, Table S5, for HDPE, PP, PS, PET. LDPE would be 0.756.
+  MWh_elec_per_tHVC_chemical_recycling: 6.9 # Material Economics (2019), page 125; based on pyrolysis and electric steam cracking
+  chlorine_production_today: 9.58 # MtCl/a from DECHEMA (2017), Table 7, page 43
+  MWh_elec_per_tCl: 3.6 # DECHEMA (2017), Table 6, page 43
+  MWh_H2_per_tCl: -0.9372  # DECHEMA (2017), page 43; negative since hydrogen produced in chloralkali process
+  methanol_production_today: 1.5 # MtMeOH/a from DECHEMA (2017), page 62
+  MWh_elec_per_tMeOH: 0.167 # DECHEMA (2017), Table 14, page 65
+  MWh_CH4_per_tMeOH: 10.25 # DECHEMA (2017), Table 14, page 65
+  hotmaps_locate_missing: false
+  reference_year: 2015
+  # references:
+  # DECHEMA (2017): https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf
+  # Material Economics (2019): https://materialeconomics.com/latest-updates/industrial-transformation-2050
+
+costs:
+  year: 2030
+  version: v0.5.0
+  rooftop_share: 0.14  # based on the potentials, assuming  (0.1 kW/m2 and 10 m2/person)
+  fill_values:
+    FOM: 0
+    VOM: 0
+    efficiency: 1
+    fuel: 0
+    investment: 0
+    lifetime: 25
+    "CO2 intensity": 0
+    "discount rate": 0.07
+  # Marginal and capital costs can be overwritten
+  # capital_cost:
+  #   onwind: 500
+  marginal_cost:
+    solar: 0.01
+    onwind: 0.015
+    offwind: 0.015
+    hydro: 0.
+    H2: 0.
+    electrolysis: 0.
+    fuel cell: 0.
+    battery: 0.
+    battery inverter: 0.
+  emission_prices: # in currency per tonne emission, only used with the option Ep
+    co2: 0.
+
+clustering:
+  simplify_network:
+    to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections)
+    algorithm: kmeans # choose from: [hac, kmeans]
+    feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc.
+    exclude_carriers: []
+    remove_stubs: true
+    remove_stubs_across_borders: true
+  cluster_network:
+    algorithm: kmeans
+    feature: solar+onwind-time
+    exclude_carriers: []
+  aggregation_strategies:
+    generators:
+      p_nom_max: sum # use "min" for more conservative assumptions
+      p_nom_min: sum
+      p_min_pu: mean
+      marginal_cost: mean
+      committable: any
+      ramp_limit_up: max
+      ramp_limit_down: max
+      efficiency: mean
+
+solving:
+  #tmpdir: "path/to/tmp"
+  options:
+    formulation: kirchhoff
+    clip_p_max_pu: 1.e-2
+    load_shedding: false
+    noisy_costs: true
+    skip_iterations: true
+    track_iterations: false
+    min_iterations: 4
+    max_iterations: 6
+    seed: 123
+
+  solver:
+    name: gurobi
+    options: gurobi-default
+
+  solver_options:
+    highs-default:
+      # refer to https://ergo-code.github.io/HiGHS/options/definitions.html#solver
+      threads: 4
+      solver: "ipm"
+      run_crossover: "off"
+      small_matrix_value: 1e-6
+      large_matrix_value: 1e9
+      primal_feasibility_tolerance: 1e-5
+      dual_feasibility_tolerance: 1e-5
+      ipm_optimality_tolerance: 1e-4
+      parallel: "on"
+      random_seed: 123
+    gurobi-default:
+      threads: 4
+      method: 2 # barrier
+      crossover: 0
+      BarConvTol: 1.e-6
+      Seed: 123
+      AggFill: 0
+      PreDual: 0
+      GURO_PAR_BARDENSETHRESH: 200
+      seed: 10              # Consistent seed for all plattforms
+    gurobi-numeric-focus:
+      name: gurobi
+      NumericFocus: 3       # Favour numeric stability over speed
+      method: 2             # barrier
+      crossover: 0          # do not use crossover
+      BarHomogeneous: 1     # Use homogeneous barrier if standard does not converge
+      BarConvTol: 1.e-5
+      FeasibilityTol: 1.e-4
+      OptimalityTol: 1.e-4
+      ObjScale: -0.5
+      threads: 8
+      Seed: 123
+    gurobi-fallback:        # Use gurobi defaults
+      name: gurobi
+      crossover: 0
+      method: 2             # barrier
+      BarHomogeneous: 1     # Use homogeneous barrier if standard does not converge
+      BarConvTol: 1.e-5
+      FeasibilityTol: 1.e-5
+      OptimalityTol: 1.e-5
+      Seed: 123
+      threads: 8
+    cplex-default:
+      threads: 4
+      lpmethod: 4 # barrier
+      solutiontype: 2 # non basic solution, ie no crossover
+      barrier.convergetol: 1.e-5
+      feasopt.tolerance: 1.e-6
+    cbc-default: {} # Used in CI
+    glpk-default: {} # Used in CI
+
+  mem: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2
+
+
+plotting:
+  map:
+    boundaries: [-11, 30, 34, 71]
+    color_geomap:
+      ocean: white
+      land: white
+  eu_node_location:
+    x: -5.5
+    y: 46.
+  costs_max: 1000
+  costs_threshold: 1
+  energy_max: 20000
+  energy_min: -20000
+  energy_threshold: 50.
+  vre_techs:
+  - onwind
+  - offwind-ac
+  - offwind-dc
+  - solar
+  - ror
+  renewable_storage_techs:
+  - PHS
+  - hydro
+  conv_techs:
+  - OCGT
+  - CCGT
+  - Nuclear
+  - Coal
+  storage_techs:
+  - hydro+PHS
+  - battery
+  - H2
+  load_carriers:
+  - AC load
+  AC_carriers:
+  - AC line
+  - AC transformer
+  link_carriers:
+  - DC line
+  - Converter AC-DC
+  heat_links:
+  - heat pump
+  - resistive heater
+  - CHP heat
+  - CHP electric
+  - gas boiler
+  - central heat pump
+  - central resistive heater
+  - central CHP heat
+  - central CHP electric
+  - central gas boiler
+  heat_generators:
+  - gas boiler
+  - central gas boiler
+  - solar thermal collector
+  - central solar thermal collector
+
+  nice_names:
+    OCGT: "Open-Cycle Gas"
+    CCGT: "Combined-Cycle Gas"
+    offwind-ac: "Offshore Wind (AC)"
+    offwind-dc: "Offshore Wind (DC)"
+    onwind: "Onshore Wind"
+    solar: "Solar"
+    PHS: "Pumped Hydro Storage"
+    hydro: "Reservoir & Dam"
+    battery: "Battery Storage"
+    H2: "Hydrogen Storage"
+    lines: "Transmission Lines"
+    ror: "Run of River"
+
+  tech_colors:
+    # wind
+    onwind: "#235ebc"
+    onshore wind: "#235ebc"
+    offwind: "#6895dd"
+    offshore wind: "#6895dd"
+    offwind-ac: "#6895dd"
+    offshore wind (AC): "#6895dd"
+    offshore wind ac: "#6895dd"
+    offwind-dc: "#74c6f2"
+    offshore wind (DC): "#74c6f2"
+    offshore wind dc: "#74c6f2"
+    # water
+    hydro: '#298c81'
+    hydro reservoir: '#298c81'
+    ror: '#3dbfb0'
+    run of river: '#3dbfb0'
+    hydroelectricity: '#298c81'
+    PHS: '#51dbcc'
+    hydro+PHS: "#08ad97"
+    wave: '#a7d4cf'
+    # solar
+    solar: "#f9d002"
+    solar PV: "#f9d002"
+    solar thermal: '#ffbf2b'
+    solar rooftop: '#ffea80'
+    # gas
+    OCGT: '#e0986c'
+    OCGT marginal: '#e0986c'
+    OCGT-heat: '#e0986c'
+    gas boiler: '#db6a25'
+    gas boilers: '#db6a25'
+    gas boiler marginal: '#db6a25'
+    gas: '#e05b09'
+    fossil gas: '#e05b09'
+    natural gas: '#e05b09'
+    CCGT: '#a85522'
+    CCGT marginal: '#a85522'
+    allam: '#B98F76'
+    gas for industry co2 to atmosphere: '#692e0a'
+    gas for industry co2 to stored: '#8a3400'
+    gas for industry: '#853403'
+    gas for industry CC: '#692e0a'
+    gas pipeline: '#ebbca0'
+    gas pipeline new: '#a87c62'
+    # oil
+    oil: '#c9c9c9'
+    oil boiler: '#adadad'
+    agriculture machinery oil: '#949494'
+    shipping oil: "#808080"
+    land transport oil: '#afafaf'
+    # nuclear
+    Nuclear: '#ff8c00'
+    Nuclear marginal: '#ff8c00'
+    nuclear: '#ff8c00'
+    uranium: '#ff8c00'
+    # coal
+    Coal: '#545454'
+    coal: '#545454'
+    Coal marginal: '#545454'
+    solid: '#545454'
+    Lignite: '#826837'
+    lignite: '#826837'
+    Lignite marginal: '#826837'
+    # biomass
+    biogas: '#e3d37d'
+    biomass: '#baa741'
+    solid biomass: '#baa741'
+    solid biomass transport: '#baa741'
+    solid biomass for industry: '#7a6d26'
+    solid biomass for industry CC: '#47411c'
+    solid biomass for industry co2 from atmosphere: '#736412'
+    solid biomass for industry co2 to stored: '#47411c'
+    biomass boiler: '#8A9A5B'
+    biomass to liquid: '#32CD32'
+    BioSNG: '#123456'
+    # power transmission
+    lines: '#6c9459'
+    transmission lines: '#6c9459'
+    electricity distribution grid: '#97ad8c'
+    # electricity demand
+    Electric load: '#110d63'
+    electric demand: '#110d63'
+    electricity: '#110d63'
+    industry electricity: '#2d2a66'
+    industry new electricity: '#2d2a66'
+    agriculture electricity: '#494778'
+    # battery + EVs
+    battery: '#ace37f'
+    battery storage: '#ace37f'
+    home battery: '#80c944'
+    home battery storage: '#80c944'
+    BEV charger: '#baf238'
+    V2G: '#e5ffa8'
+    land transport EV: '#baf238'
+    Li ion: '#baf238'
+    # hot water storage
+    water tanks: '#e69487'
+    hot water storage: '#e69487'
+    hot water charging: '#e69487'
+    hot water discharging: '#e69487'
+    # heat demand
+    Heat load: '#cc1f1f'
+    heat: '#cc1f1f'
+    heat demand: '#cc1f1f'
+    rural heat: '#ff5c5c'
+    central heat: '#cc1f1f'
+    decentral heat: '#750606'
+    low-temperature heat for industry: '#8f2727'
+    process heat: '#ff0000'
+    agriculture heat: '#d9a5a5'
+    # heat supply
+    heat pumps: '#2fb537'
+    heat pump: '#2fb537'
+    air heat pump: '#36eb41'
+    ground heat pump: '#2fb537'
+    Ambient: '#98eb9d'
+    CHP: '#8a5751'
+    CHP CC: '#634643'
+    CHP heat: '#8a5751'
+    CHP electric: '#8a5751'
+    district heating: '#e8beac'
+    resistive heater: '#d8f9b8'
+    retrofitting: '#8487e8'
+    building retrofitting: '#8487e8'
+    # hydrogen
+    H2 for industry: "#f073da"
+    H2 for shipping: "#ebaee0"
+    H2: '#bf13a0'
+    hydrogen: '#bf13a0'
+    SMR: '#870c71'
+    SMR CC: '#4f1745'
+    H2 liquefaction: '#d647bd'
+    hydrogen storage: '#bf13a0'
+    H2 storage: '#bf13a0'
+    land transport fuel cell: '#6b3161'
+    H2 pipeline: '#f081dc'
+    H2 pipeline retrofitted: '#ba99b5'
+    H2 Fuel Cell: '#c251ae'
+    H2 Electrolysis: '#ff29d9'
+    # ammonia
+    NH3: '#46caf0'
+    ammonia: '#46caf0'
+    ammonia store: '#00ace0'
+    ammonia cracker: '#87d0e6'
+    Haber-Bosch: '#076987'
+    # syngas
+    Sabatier: '#9850ad'
+    methanation: '#c44ce6'
+    methane: '#c44ce6'
+    helmeth: '#e899ff'
+    # synfuels
+    Fischer-Tropsch: '#25c49a'
+    liquid: '#25c49a'
+    kerosene for aviation: '#a1ffe6'
+    naphtha for industry: '#57ebc4'
+    methanolisation: '#83d6d5'
+    methanol: '#468c8b'
+    shipping methanol: '#468c8b'
+    # co2
+    CC: '#f29dae'
+    CCS: '#f29dae'
+    CO2 sequestration: '#f29dae'
+    DAC: '#ff5270'
+    co2 stored: '#f2385a'
+    co2: '#f29dae'
+    co2 vent: '#ffd4dc'
+    CO2 pipeline: '#f5627f'
+    # emissions
+    process emissions CC: '#000000'
+    process emissions: '#222222'
+    process emissions to stored: '#444444'
+    process emissions to atmosphere: '#888888'
+    oil emissions: '#aaaaaa'
+    shipping oil emissions: "#555555"
+    shipping methanol emissions: '#666666'
+    land transport oil emissions: '#777777'
+    agriculture machinery oil emissions: '#333333'
+    # other
+    shipping: '#03a2ff'
+    power-to-heat: '#2fb537'
+    power-to-gas: '#c44ce6'
+    power-to-H2: '#ff29d9'
+    power-to-liquid: '#25c49a'
+    gas-to-power/heat: '#ee8340'
+    waste: '#e3d37d'
+    other: '#000000'
+    geothermal: '#ba91b1'
+    AC-AC: "#70af1d"
+    AC line: "#70af1d"
+    links: "#8a1caf"
+    HVDC links: "#8a1caf"
+    DC-DC: "#8a1caf"
+    DC link: "#8a1caf"
diff --git a/configs/test/config.electricity.yaml b/configs/test/config.electricity.yaml
new file mode 100644
index 00000000..6798e38c
--- /dev/null
+++ b/configs/test/config.electricity.yaml
@@ -0,0 +1,80 @@
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: CC0-1.0
+
+tutorial: true
+
+
+run:
+  name: "test-elec" # use this to keep track of runs with different settings
+  disable_progressbar: true
+  shared_resources: true
+  shared_cutouts: true
+
+scenario:
+  clusters:
+  - 5
+  opts:
+  - Co2L-24H
+
+countries: ['BE']
+
+snapshots:
+  start: "2013-03-01"
+  end: "2013-03-08"
+
+electricity:
+  co2limit: 100.e+6
+
+  extendable_carriers:
+    Generator: [OCGT]
+    StorageUnit: [battery]
+    Store: [H2]
+    Link: [H2 pipeline]
+
+  renewable_carriers: [solar, onwind, offwind-ac, offwind-dc]
+
+
+atlite:
+  default_cutout: be-03-2013-era5
+  cutouts:
+    be-03-2013-era5:
+      module: era5
+      x: [4., 15.]
+      y: [46., 56.]
+      time: ["2013-03-01", "2013-03-08"]
+
+renewable:
+  onwind:
+    cutout: be-03-2013-era5
+  offwind-ac:
+    cutout: be-03-2013-era5
+    max_depth: false
+  offwind-dc:
+    cutout: be-03-2013-era5
+    max_depth: false
+  solar:
+    cutout: be-03-2013-era5
+
+
+clustering:
+  exclude_carriers: ["OCGT", "offwind-ac", "coal"]
+
+
+solving:
+  solver:
+    name: glpk
+    options: "glpk-default"
+
+
+plotting:
+  map:
+    boundaries:
+  eu_node_location:
+    x: -5.5
+    y: 46.
+  costs_max: 1000
+  costs_threshold: 0.0000001
+  energy_max:
+  energy_min:
+  energy_threshold: 0.000001
diff --git a/configs/test/config.myopic.yaml b/configs/test/config.myopic.yaml
new file mode 100644
index 00000000..efa03136
--- /dev/null
+++ b/configs/test/config.myopic.yaml
@@ -0,0 +1,79 @@
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: CC0-1.0
+
+tutorial: true
+
+run:
+  name: "test-sector-myopic"
+  disable_progressbar: true
+  shared_resources: true
+  shared_cutouts: true
+
+foresight: myopic
+
+scenario:
+  ll:
+  - v1.5
+  clusters:
+  - 5
+  sector_opts:
+  - 24H-T-H-B-I-A-solar+p3-dist1
+  planning_horizons:
+  - 2030
+  - 2040
+  - 2050
+
+countries: ['BE']
+
+snapshots:
+  start: "2013-03-01"
+  end: "2013-03-08"
+
+electricity:
+  renewable_carriers: [solar, onwind, offwind-ac, offwind-dc]
+
+atlite:
+  default_cutout: be-03-2013-era5
+  cutouts:
+    be-03-2013-era5:
+      module: era5
+      x: [4., 15.]
+      y: [46., 56.]
+      time: ["2013-03-01", "2013-03-08"]
+
+renewable:
+  onwind:
+    cutout: be-03-2013-era5
+  offwind-ac:
+    cutout: be-03-2013-era5
+    max_depth: false
+  offwind-dc:
+    cutout: be-03-2013-era5
+    max_depth: false
+  solar:
+    cutout: be-03-2013-era5
+
+industry:
+  St_primary_fraction:
+    2030: 0.6
+    2040: 0.5
+    2050: 0.4
+
+solving:
+  solver:
+    name: glpk
+    options: glpk-default
+  mem: 4000
+
+plotting:
+  map:
+    boundaries:
+  eu_node_location:
+    x: -5.5
+    y: 46.
+  costs_max: 1000
+  costs_threshold: 0.0000001
+  energy_max:
+  energy_min:
+  energy_threshold: 0.000001
diff --git a/configs/test/config.overnight.yaml b/configs/test/config.overnight.yaml
new file mode 100644
index 00000000..fb468ded
--- /dev/null
+++ b/configs/test/config.overnight.yaml
@@ -0,0 +1,74 @@
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: CC0-1.0
+
+tutorial: true
+
+run:
+  name: "test-sector-overnight"
+  disable_progressbar: true
+  shared_resources: true
+  shared_cutouts: true
+
+
+scenario:
+  ll:
+  - v1.5
+  clusters:
+  - 5
+  sector_opts:
+  - CO2L0-24H-T-H-B-I-A-solar+p3-dist1
+  planning_horizons:
+  - 2030
+
+countries: ['BE']
+
+snapshots:
+  start: "2013-03-01"
+  end: "2013-03-08"
+
+electricity:
+  renewable_carriers: [solar, onwind, offwind-ac, offwind-dc]
+
+atlite:
+  default_cutout: be-03-2013-era5
+  cutouts:
+    be-03-2013-era5:
+      module: era5
+      x: [4., 15.]
+      y: [46., 56.]
+      time: ["2013-03-01", "2013-03-08"]
+
+renewable:
+  onwind:
+    cutout: be-03-2013-era5
+  offwind-ac:
+    cutout: be-03-2013-era5
+    max_depth: false
+  offwind-dc:
+    cutout: be-03-2013-era5
+    max_depth: false
+  solar:
+    cutout: be-03-2013-era5
+
+sector:
+  gas_network: true
+  H2_retrofit: true
+
+solving:
+  solver:
+    name: glpk
+    options: glpk-default
+  mem: 4000
+
+plotting:
+  map:
+    boundaries:
+  eu_node_location:
+    x: -5.5
+    y: 46.
+  costs_max: 1000
+  costs_threshold: 0.0000001
+  energy_max:
+  energy_min:
+  energy_threshold: 0.000001

From 690ccba43f78d7929eb2489da9c27e0d6ca85ed0 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Tue, 11 Apr 2023 02:13:21 +0200
Subject: [PATCH 006/176] Delete config.default.yaml

---
 config.default.yaml | 959 --------------------------------------------
 1 file changed, 959 deletions(-)
 delete mode 100755 config.default.yaml

diff --git a/config.default.yaml b/config.default.yaml
deleted file mode 100755
index 5218bc22..00000000
--- a/config.default.yaml
+++ /dev/null
@@ -1,959 +0,0 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
-#
-# SPDX-License-Identifier: CC0-1.0
-
-version: 0.8.0
-tutorial: false
-
-logging:
-  level: INFO
-  format: '%(levelname)s:%(name)s:%(message)s'
-
-run:
-  name: "" # use this to keep track of runs with different settings
-  disable_progressbar: false # set to true to disable the progressbar
-  shared_resources: false # set to true to share the default resources across runs
-  shared_cutouts: true # set to true to share the default cutout(s) across runs
-
-foresight: overnight # options are overnight, myopic, perfect (perfect is not yet implemented)
-# if you use myopic or perfect foresight, set the investment years in "planning_horizons" below
-
-scenario:
-  simpl:
-  - ''
-  ll:  # allowed transmission line volume expansion, can be any float >= 1.0 with a prefix v|c (today) or "copt"
-  - v1.0
-  - v1.5
-  clusters:  # number of nodes in Europe, any integer between 37 (1 node per country-zone) and several hundred
-  - 37
-  - 128
-  - 256
-  - 512
-  - 1024
-  opts: # only relevant for PyPSA-Eur
-  - ''
-  sector_opts: # this is where the main scenario settings are
-  - Co2L0-3H-T-H-B-I-A-solar+p3-dist1
-  # to really understand the options here, look in scripts/prepare_sector_network.py
-  # Co2Lx specifies the CO2 target in x% of the 1990 values; default will give default (5%);
-  # Co2L0p25 will give 25% CO2 emissions; Co2Lm0p05 will give 5% negative emissions
-  # xH is the temporal resolution; 3H is 3-hourly, i.e. one snapshot every 3 hours
-  # single letters are sectors: T for land transport, H for building heating,
-  # B for biomass supply, I for industry, shipping and aviation,
-  # A for agriculture, forestry and fishing
-  # solar+c0.5 reduces the capital cost of solar to 50\% of reference value
-  # solar+p3 multiplies the available installable potential by factor 3
-  # seq400 sets the potential of CO2 sequestration to 400 Mt CO2 per year
-  # dist{n} includes distribution grids with investment cost of n times cost in data/costs.csv
-  # for myopic/perfect foresight cb states the carbon budget in GtCO2 (cumulative
-  # emissions throughout the transition path in the timeframe determined by the
-  # planning_horizons), be:beta decay; ex:exponential decay
-  # cb40ex0 distributes a carbon budget of 40 GtCO2 following an exponential
-  # decay with initial growth rate 0
-  planning_horizons: # investment years for myopic and perfect; for overnight, year of cost assumptions can be different and is defined under 'costs'
-  - 2050
-  # for example, set to
-  # - 2020
-  # - 2030
-  # - 2040
-  # - 2050
-  # for myopic foresight
-
-countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK']
-
-snapshots:
-  start: "2013-01-01"
-  end: "2014-01-01"
-  inclusive: 'left' # include start, not end
-
-enable:
-  prepare_links_p_nom: false
-  retrieve_databundle: true
-  retrieve_sector_databundle: true
-  retrieve_cost_data: true
-  build_cutout: false
-  retrieve_cutout: true
-  build_natura_raster: false
-  retrieve_natura_raster: true
-  custom_busmap: false
-
-# CO2 budget as a fraction of 1990 emissions
-# this is over-ridden if CO2Lx is set in sector_opts
-# this is also over-ridden if cb is set in sector_opts
-co2_budget:
-  2020: 0.701
-  2025: 0.524
-  2030: 0.297
-  2035: 0.150
-  2040: 0.071
-  2045: 0.032
-  2050: 0.000
-
-electricity:
-  voltages: [220., 300., 380.]
-  gaslimit: false # global gas usage limit of X MWh_th
-  co2limit: 7.75e+7 # 0.05 * 3.1e9*0.5
-  co2base: 1.487e+9
-  agg_p_nom_limits: data/agg_p_nom_minmax.csv
-
-  operational_reserve: # like https://genxproject.github.io/GenX/dev/core/#Reserves
-    activate: false
-    epsilon_load: 0.02 # share of total load
-    epsilon_vres: 0.02 # share of total renewable supply
-    contingency: 4000 # fixed capacity in MW
-
-  max_hours:
-    battery: 6
-    H2: 168
-
-  extendable_carriers:
-    Generator: [solar, onwind, offwind-ac, offwind-dc, OCGT]
-    StorageUnit: [] # battery, H2
-    Store: [battery, H2]
-    Link: [] # H2 pipeline
-
-  # use pandas query strings here, e.g. Country not in ['Germany']
-  powerplants_filter: (DateOut >= 2022 or DateOut != DateOut)
-  # use pandas query strings here, e.g. Country in ['Germany']
-  custom_powerplants: false
-
-  conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass]
-  renewable_carriers: [solar, onwind, offwind-ac, offwind-dc, hydro]
-
-  estimate_renewable_capacities:
-    enable: true
-    # Add capacities from OPSD data
-    from_opsd: true
-    # Renewable capacities are based on existing capacities reported by IRENA
-    year: 2020
-    # Artificially limit maximum capacities to factor * (IRENA capacities),
-    # i.e. 110% of <years>'s capacities => expansion_limit: 1.1
-    # false: Use estimated renewable potentials determine by the workflow
-    expansion_limit: false
-    technology_mapping:
-      # Wind is the Fueltype in powerplantmatching, onwind, offwind-{ac,dc} the carrier in PyPSA-Eur
-      Offshore: [offwind-ac, offwind-dc]
-      Onshore: [onwind]
-      PV: [solar]
-
-
-atlite:
-  default_cutout: europe-2013-era5
-  nprocesses: 4
-  show_progress: false # false saves time
-  cutouts:
-    # use 'base' to determine geographical bounds and time span from config
-    # base:
-      # module: era5
-    europe-2013-era5:
-      module: era5 # in priority order
-      x: [-12., 35.]
-      y: [33., 72]
-      dx: 0.3
-      dy: 0.3
-      time: ['2013', '2013']
-    europe-2013-sarah:
-      module: [sarah, era5] # in priority order
-      x: [-12., 45.]
-      y: [33., 65]
-      dx: 0.2
-      dy: 0.2
-      time: ['2013', '2013']
-      sarah_interpolate: false
-      sarah_dir:
-      features: [influx, temperature]
-
-
-renewable:
-  onwind:
-    cutout: europe-2013-era5
-    resource:
-      method: wind
-      turbine: Vestas_V112_3MW
-    capacity_per_sqkm: 3 # ScholzPhd Tab 4.3.1: 10MW/km^2 and assuming 30% fraction of the already restricted
-    # area is available for installation of wind generators due to competing land use and likely public
-    # acceptance issues.
-    # correction_factor: 0.93
-    corine:
-      # Scholz, Y. (2012). Renewable energy based electricity supply at low costs
-      #  development of the REMix model and application for Europe. ( p.42 / p.28)
-      grid_codes: [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32]
-      distance: 1000
-      distance_grid_codes: [1, 2, 3, 4, 5, 6]
-    natura: true
-    excluder_resolution: 100
-    potential: simple # or conservative
-    clip_p_max_pu: 1.e-2
-  offwind-ac:
-    cutout: europe-2013-era5
-    resource:
-      method: wind
-      turbine: NREL_ReferenceTurbine_5MW_offshore
-    capacity_per_sqkm: 2 # ScholzPhd Tab 4.3.1: 10MW/km^2 and assuming 20% fraction of the already restricted
-    # area is available for installation of wind generators due to competing land use and likely public
-    # acceptance issues.
-    correction_factor: 0.8855
-    # proxy for wake losses
-    # from 10.1016/j.energy.2018.08.153
-    # until done more rigorously in #153
-    corine: [44, 255]
-    natura: true
-    ship_threshold: 400
-    max_depth: 50
-    max_shore_distance: 30000
-    excluder_resolution: 200
-    potential: simple # or conservative
-    clip_p_max_pu: 1.e-2
-  offwind-dc:
-    cutout: europe-2013-era5
-    resource:
-      method: wind
-      turbine: NREL_ReferenceTurbine_5MW_offshore
-    capacity_per_sqkm: 2 # ScholzPhd Tab 4.3.1: 10MW/km^2 and assuming 20% fraction of the already restricted
-    # area is available for installation of wind generators due to competing land use and likely public
-    # acceptance issues.
-    correction_factor: 0.8855
-    # proxy for wake losses
-    # from 10.1016/j.energy.2018.08.153
-    # until done more rigorously in #153
-    corine: [44, 255]
-    natura: true
-    ship_threshold: 400
-    max_depth: 50
-    min_shore_distance: 30000
-    excluder_resolution: 200
-    potential: simple # or conservative
-    clip_p_max_pu: 1.e-2
-  solar:
-    cutout: europe-2013-sarah
-    resource:
-      method: pv
-      panel: CSi
-      orientation:
-        slope: 35.
-        azimuth: 180.
-    capacity_per_sqkm: 1.7 # ScholzPhd Tab 4.3.1: 170 MW/km^2 and assuming 1% of the area can be used for solar PV panels
-    # Correction factor determined by comparing uncorrected area-weighted full-load hours to those
-    # published in Supplementary Data to
-    # Pietzcker, Robert Carl, et al. "Using the sun to decarbonize the power
-    # sector -- The economic potential of photovoltaics and concentrating solar
-    # power." Applied Energy 135 (2014): 704-720.
-    # This correction factor of 0.854337 may be in order if using reanalysis data.
-    # for discussion refer to https://github.com/PyPSA/pypsa-eur/pull/304
-    # correction_factor: 0.854337
-    corine: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 26, 31, 32]
-    natura: true
-    excluder_resolution: 100
-    potential: simple # or conservative
-    clip_p_max_pu: 1.e-2
-  hydro:
-    cutout: europe-2013-era5
-    carriers: [ror, PHS, hydro]
-    PHS_max_hours: 6
-    hydro_max_hours: "energy_capacity_totals_by_country" # one of energy_capacity_totals_by_country, estimate_by_large_installations or a float
-    clip_min_inflow: 1.0
-
-conventional:
-  nuclear:
-    p_max_pu: "data/nuclear_p_max_pu.csv" # float of file name
-
-lines:
-  types:
-    220.: "Al/St 240/40 2-bundle 220.0"
-    300.: "Al/St 240/40 3-bundle 300.0"
-    380.: "Al/St 240/40 4-bundle 380.0"
-  s_max_pu: 0.7
-  s_nom_max: .inf
-  length_factor: 1.25
-  under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity
-
-links:
-  p_max_pu: 1.0
-  p_nom_max: .inf
-  include_tyndp: true
-  under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity
-
-transformers:
-  x: 0.1
-  s_nom: 2000.
-  type: ''
-
-load:
-  power_statistics: true # only for files from <2019; set false in order to get ENTSOE transparency data
-  interpolate_limit: 3 # data gaps up until this size are interpolated linearly
-  time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from
-  manual_adjustments: true # false
-  scaling_factor: 1.0
-
-# regulate what components with which carriers are kept from PyPSA-Eur;
-# some technologies are removed because they are implemented differently
-# (e.g. battery or H2 storage) or have different year-dependent costs
-# in PyPSA-Eur-Sec
-pypsa_eur:
-  Bus:
-  - AC
-  Link:
-  - DC
-  Generator:
-  - onwind
-  - offwind-ac
-  - offwind-dc
-  - solar
-  - ror
-  StorageUnit:
-  - PHS
-  - hydro
-  Store: []
-
-energy:
-  energy_totals_year: 2011
-  base_emissions_year: 1990
-  eurostat_report_year: 2016
-  emissions: CO2 # "CO2" or "All greenhouse gases - (CO2 equivalent)"
-
-biomass:
-  year: 2030
-  scenario: ENS_Med
-  classes:
-    solid biomass:
-    - Agricultural waste
-    - Fuelwood residues
-    - Secondary Forestry residues - woodchips
-    - Sawdust
-    - Residues from landscape care
-    - Municipal waste
-    not included:
-    - Sugar from sugar beet
-    - Rape seed
-    - "Sunflower, soya seed "
-    - Bioethanol barley, wheat, grain maize, oats, other cereals and rye
-    - Miscanthus, switchgrass, RCG
-    - Willow
-    - Poplar
-    - FuelwoodRW
-    - C&P_RW
-    biogas:
-    - Manure solid, liquid
-    - Sludge
-
-
-solar_thermal:
-  clearsky_model: simple  # should be "simple" or "enhanced"?
-  orientation:
-    slope: 45.
-    azimuth: 180.
-
-# only relevant for foresight = myopic or perfect
-existing_capacities:
-  grouping_years_power: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030]
-  grouping_years_heat: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019] # these should not extend 2020
-  threshold_capacity: 10
-  conventional_carriers:
-  - lignite
-  - coal
-  - oil
-  - uranium
-
-
-sector:
-  district_heating:
-    potential: 0.6  # maximum fraction of urban demand which can be supplied by district heating
-     # increase of today's district heating demand to potential maximum district heating share
-     # progress = 0 means today's district heating share, progress = 1 means maximum fraction of urban demand is supplied by district heating
-    progress:
-      2020: 0.0
-      2030: 0.3
-      2040: 0.6
-      2050: 1.0
-    district_heating_loss: 0.15
-  cluster_heat_buses: false # cluster residential and service heat buses to one to save memory
-  bev_dsm_restriction_value: 0.75  #Set to 0 for no restriction on BEV DSM
-  bev_dsm_restriction_time: 7  #Time at which SOC of BEV has to be dsm_restriction_value
-  transport_heating_deadband_upper: 20.
-  transport_heating_deadband_lower: 15.
-  ICE_lower_degree_factor: 0.375  #in per cent increase in fuel consumption per degree above deadband
-  ICE_upper_degree_factor: 1.6
-  EV_lower_degree_factor: 0.98
-  EV_upper_degree_factor: 0.63
-  bev_dsm: true #turns on EV battery
-  bev_availability: 0.5  #How many cars do smart charging
-  bev_energy: 0.05  #average battery size in MWh
-  bev_charge_efficiency: 0.9  #BEV (dis-)charging efficiency
-  bev_plug_to_wheel_efficiency: 0.2 #kWh/km from EPA https://www.fueleconomy.gov/feg/ for Tesla Model S
-  bev_charge_rate: 0.011 #3-phase charger with 11 kW
-  bev_avail_max: 0.95
-  bev_avail_mean: 0.8
-  v2g: true #allows feed-in to grid from EV battery
-  #what is not EV or FCEV is oil-fuelled ICE
-  land_transport_fuel_cell_share:
-    2020: 0
-    2030: 0.05
-    2040: 0.1
-    2050: 0.15
-  land_transport_electric_share:
-    2020: 0
-    2030: 0.25
-    2040: 0.6
-    2050: 0.85
-  land_transport_ice_share:
-    2020: 1
-    2030: 0.7
-    2040: 0.3
-    2050: 0
-  transport_fuel_cell_efficiency: 0.5
-  transport_internal_combustion_efficiency: 0.3
-  agriculture_machinery_electric_share: 0
-  agriculture_machinery_oil_share: 1
-  agriculture_machinery_fuel_efficiency: 0.7 # fuel oil per use
-  agriculture_machinery_electric_efficiency: 0.3 # electricity per use
-  MWh_MeOH_per_MWh_H2: 0.8787 # in LHV, source: DECHEMA (2017): Low carbon energy and feedstock for the European chemical industry , pg. 64.
-  MWh_MeOH_per_tCO2: 4.0321 # in LHV, source: DECHEMA (2017): Low carbon energy and feedstock for the European chemical industry , pg. 64.
-  MWh_MeOH_per_MWh_e: 3.6907 # in LHV, source: DECHEMA (2017): Low carbon energy and feedstock for the European chemical industry , pg. 64.
-  shipping_hydrogen_liquefaction: false # whether to consider liquefaction costs for shipping H2 demands
-  shipping_hydrogen_share:
-    2020: 0
-    2030: 0
-    2040: 0
-    2050: 0
-  shipping_methanol_share:
-    2020: 0
-    2030: 0.3
-    2040: 0.7
-    2050: 1
-  shipping_oil_share:
-    2020: 1
-    2030: 0.7
-    2040: 0.3
-    2050: 0
-  shipping_methanol_efficiency: 0.46 # 10-15% higher https://www.iea-amf.org/app/webroot/files/file/Annex%20Reports/AMF_Annex_56.pdf, https://users.ugent.be/~lsileghe/documents/extended_abstract.pdf
-  shipping_oil_efficiency: 0.40 #For conversion of fuel oil to propulsion in 2011
-  aviation_demand_factor: 1. # relative aviation demand compared to today
-  HVC_demand_factor: 1. # relative HVC demand compared to today
-  time_dep_hp_cop: true #time dependent heat pump coefficient of performance
-  heat_pump_sink_T: 55. # Celsius, based on DTU / large area radiators; used in build_cop_profiles.py
-   # conservatively high to cover hot water and space heating in poorly-insulated buildings
-  reduce_space_heat_exogenously: true  # reduces space heat demand by a given factor (applied before losses in DH)
-  # this can represent e.g. building renovation, building demolition, or if
-  # the factor is negative: increasing floor area, increased thermal comfort, population growth
-  reduce_space_heat_exogenously_factor:  # per unit reduction in space heat demand
-  # the default factors are determined by the LTS scenario from http://tool.european-calculator.eu/app/buildings/building-types-area/?levers=1ddd4444421213bdbbbddd44444ffffff11f411111221111211l212221
-    2020: 0.10  # this results in a space heat demand reduction of 10%
-    2025: 0.09  # first heat demand increases compared to 2020 because of larger floor area per capita
-    2030: 0.09
-    2035: 0.11
-    2040: 0.16
-    2045: 0.21
-    2050: 0.29
-  retrofitting:   # co-optimises building renovation to reduce space heat demand
-    retro_endogen: false  # co-optimise space heat savings
-    cost_factor: 1.0   # weight costs for building renovation
-    interest_rate: 0.04  # for investment in building components
-    annualise_cost: true  # annualise the investment costs
-    tax_weighting: false   # weight costs depending on taxes in countries
-    construction_index: true   # weight costs depending on labour/material costs per country
-  tes: true
-  tes_tau: # 180 day time constant for centralised, 3 day for decentralised
-    decentral: 3
-    central: 180
-  boilers: true
-  oil_boilers: false
-  biomass_boiler: true
-  chp: true
-  micro_chp: false
-  solar_thermal: true
-  solar_cf_correction: 0.788457  # =  >>> 1/1.2683
-  marginal_cost_storage: 0. #1e-4
-  methanation: true
-  helmeth: true
-  coal_cc: false
-  dac: true
-  co2_vent: false
-  allam_cycle: false
-  SMR: true
-  regional_co2_sequestration_potential:
-    enable: false  # enable regionally resolved geological co2 storage potential
-    attribute: 'conservative estimate Mt'
-    include_onshore: false  # include onshore sequestration potentials
-    min_size: 3 # Gt, sites with lower potential will be excluded
-    max_size: 25 # Gt, max sequestration potential for any one site, TODO research suitable value
-    years_of_storage: 25 # years until potential exhausted at optimised annual rate
-  co2_sequestration_potential: 200  #MtCO2/a sequestration potential for Europe
-  co2_sequestration_cost: 10   #EUR/tCO2 for sequestration of CO2
-  co2_spatial: false
-  co2network: false
-  cc_fraction: 0.9  # default fraction of CO2 captured with post-combustion capture
-  hydrogen_underground_storage: true
-  hydrogen_underground_storage_locations:
-    # - onshore  # more than 50 km from sea
-  - nearshore    # within 50 km of sea
-    # - offshore
-  ammonia: false # can be false (no NH3 carrier), true (copperplated NH3), "regional" (regionalised NH3 without network)
-  min_part_load_fischer_tropsch: 0.9 # p_min_pu
-  min_part_load_methanolisation: 0.5 # p_min_pu
-  use_fischer_tropsch_waste_heat: true
-  use_fuel_cell_waste_heat: true
-  use_electrolysis_waste_heat: false
-  electricity_distribution_grid: true
-  electricity_distribution_grid_cost_factor: 1.0  #multiplies cost in data/costs.csv
-  electricity_grid_connection: true  # only applies to onshore wind and utility PV
-  H2_network: true
-  gas_network: false
-  H2_retrofit: false  # if set to True existing gas pipes can be retrofitted to H2 pipes
-  # according to hydrogen backbone strategy (April, 2020) p.15
-  # https://gasforclimate2050.eu/wp-content/uploads/2020/07/2020_European-Hydrogen-Backbone_Report.pdf
-  # 60% of original natural gas capacity could be used in cost-optimal case as H2 capacity
-  H2_retrofit_capacity_per_CH4: 0.6  # ratio for H2 capacity per original CH4 capacity of retrofitted pipelines
-  gas_network_connectivity_upgrade: 1 # https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation
-  gas_distribution_grid: true
-  gas_distribution_grid_cost_factor: 1.0  #multiplies cost in data/costs.csv
-  biomass_spatial: false  # regionally resolve biomass (e.g. potentials)
-  biomass_transport: false  # allow transport of solid biomass between nodes
-  conventional_generation: # generator : carrier
-    OCGT: gas
-  biomass_to_liquid: false
-  biosng: false
-
-industry:
-  St_primary_fraction:  # fraction of steel produced via primary route versus secondary route (scrap+EAF); today fraction is 0.6
-    2020: 0.6
-    2025: 0.55
-    2030: 0.5
-    2035: 0.45
-    2040: 0.4
-    2045: 0.35
-    2050: 0.3
-  DRI_fraction:  # fraction of the primary route converted to DRI + EAF
-    2020: 0
-    2025: 0
-    2030: 0.05
-    2035: 0.2
-    2040: 0.4
-    2045: 0.7
-    2050: 1
-  H2_DRI: 1.7   #H2 consumption in Direct Reduced Iron (DRI),  MWh_H2,LHV/ton_Steel from 51kgH2/tSt in Vogl et al (2018) doi:10.1016/j.jclepro.2018.08.279
-  elec_DRI: 0.322   #electricity consumption in Direct Reduced Iron (DRI) shaft, MWh/tSt HYBRIT brochure https://ssabwebsitecdn.azureedge.net/-/media/hybrit/files/hybrit_brochure.pdf
-  Al_primary_fraction:  # fraction of aluminium produced via the primary route versus scrap; today fraction is 0.4
-    2020: 0.4
-    2025: 0.375
-    2030: 0.35
-    2035: 0.325
-    2040: 0.3
-    2045: 0.25
-    2050: 0.2
-  MWh_NH3_per_tNH3: 5.166 # LHV
-  MWh_CH4_per_tNH3_SMR: 10.8 # 2012's demand from https://ec.europa.eu/docsroom/documents/4165/attachments/1/translations/en/renditions/pdf
-  MWh_elec_per_tNH3_SMR: 0.7 # same source, assuming 94-6% split methane-elec of total energy demand 11.5 MWh/tNH3
-  MWh_H2_per_tNH3_electrolysis: 6.5 # from https://doi.org/10.1016/j.joule.2018.04.017, around 0.197 tH2/tHN3 (>3/17 since some H2 lost and used for energy)
-  MWh_elec_per_tNH3_electrolysis: 1.17 # from https://doi.org/10.1016/j.joule.2018.04.017 Table 13 (air separation and HB)
-  MWh_NH3_per_MWh_H2_cracker: 1.46 # https://github.com/euronion/trace/blob/44a5ff8401762edbef80eff9cfe5a47c8d3c8be4/data/efficiencies.csv
-  NH3_process_emissions: 24.5 # in MtCO2/a from SMR for H2 production for NH3 from UNFCCC for 2015 for EU28
-  petrochemical_process_emissions: 25.5 # in MtCO2/a for petrochemical and other from UNFCCC for 2015 for EU28
-  HVC_primary_fraction: 1. # fraction of today's HVC produced via primary route
-  HVC_mechanical_recycling_fraction: 0. # fraction of today's HVC produced via mechanical recycling
-  HVC_chemical_recycling_fraction: 0. # fraction of today's HVC produced via chemical recycling
-  HVC_production_today: 52. # MtHVC/a from DECHEMA (2017), Figure 16, page 107; includes ethylene, propylene and BTX
-  MWh_elec_per_tHVC_mechanical_recycling: 0.547 # from SI of https://doi.org/10.1016/j.resconrec.2020.105010, Table S5, for HDPE, PP, PS, PET. LDPE would be 0.756.
-  MWh_elec_per_tHVC_chemical_recycling: 6.9 # Material Economics (2019), page 125; based on pyrolysis and electric steam cracking
-  chlorine_production_today: 9.58 # MtCl/a from DECHEMA (2017), Table 7, page 43
-  MWh_elec_per_tCl: 3.6 # DECHEMA (2017), Table 6, page 43
-  MWh_H2_per_tCl: -0.9372  # DECHEMA (2017), page 43; negative since hydrogen produced in chloralkali process
-  methanol_production_today: 1.5 # MtMeOH/a from DECHEMA (2017), page 62
-  MWh_elec_per_tMeOH: 0.167 # DECHEMA (2017), Table 14, page 65
-  MWh_CH4_per_tMeOH: 10.25 # DECHEMA (2017), Table 14, page 65
-  hotmaps_locate_missing: false
-  reference_year: 2015
-  # references:
-  # DECHEMA (2017): https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf
-  # Material Economics (2019): https://materialeconomics.com/latest-updates/industrial-transformation-2050
-
-costs:
-  year: 2030
-  version: v0.5.0
-  rooftop_share: 0.14  # based on the potentials, assuming  (0.1 kW/m2 and 10 m2/person)
-  fill_values:
-    FOM: 0
-    VOM: 0
-    efficiency: 1
-    fuel: 0
-    investment: 0
-    lifetime: 25
-    "CO2 intensity": 0
-    "discount rate": 0.07
-  # Marginal and capital costs can be overwritten
-  # capital_cost:
-  #   onwind: 500
-  marginal_cost:
-    solar: 0.01
-    onwind: 0.015
-    offwind: 0.015
-    hydro: 0.
-    H2: 0.
-    electrolysis: 0.
-    fuel cell: 0.
-    battery: 0.
-    battery inverter: 0.
-  emission_prices: # in currency per tonne emission, only used with the option Ep
-    co2: 0.
-
-clustering:
-  simplify_network:
-    to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections)
-    algorithm: kmeans # choose from: [hac, kmeans]
-    feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc.
-    exclude_carriers: []
-    remove_stubs: true
-    remove_stubs_across_borders: true
-  cluster_network:
-    algorithm: kmeans
-    feature: solar+onwind-time
-    exclude_carriers: []
-  aggregation_strategies:
-    generators:
-      p_nom_max: sum # use "min" for more conservative assumptions
-      p_nom_min: sum
-      p_min_pu: mean
-      marginal_cost: mean
-      committable: any
-      ramp_limit_up: max
-      ramp_limit_down: max
-      efficiency: mean
-
-solving:
-  #tmpdir: "path/to/tmp"
-  options:
-    formulation: kirchhoff
-    clip_p_max_pu: 1.e-2
-    load_shedding: false
-    noisy_costs: true
-    skip_iterations: true
-    track_iterations: false
-    min_iterations: 4
-    max_iterations: 6
-    seed: 123
-
-  solver:
-    name: gurobi
-    options: gurobi-default
-
-  solver_options:
-    highs-default:
-      # refer to https://ergo-code.github.io/HiGHS/options/definitions.html#solver
-      threads: 4
-      solver: "ipm"
-      run_crossover: "off"
-      small_matrix_value: 1e-6
-      large_matrix_value: 1e9
-      primal_feasibility_tolerance: 1e-5
-      dual_feasibility_tolerance: 1e-5
-      ipm_optimality_tolerance: 1e-4
-      parallel: "on"
-      random_seed: 123
-    gurobi-default:
-      threads: 4
-      method: 2 # barrier
-      crossover: 0
-      BarConvTol: 1.e-6
-      Seed: 123
-      AggFill: 0
-      PreDual: 0
-      GURO_PAR_BARDENSETHRESH: 200
-      seed: 10              # Consistent seed for all plattforms
-    gurobi-numeric-focus:
-      name: gurobi
-      NumericFocus: 3       # Favour numeric stability over speed
-      method: 2             # barrier
-      crossover: 0          # do not use crossover
-      BarHomogeneous: 1     # Use homogeneous barrier if standard does not converge
-      BarConvTol: 1.e-5
-      FeasibilityTol: 1.e-4
-      OptimalityTol: 1.e-4
-      ObjScale: -0.5
-      threads: 8
-      Seed: 123
-    gurobi-fallback:        # Use gurobi defaults
-      name: gurobi
-      crossover: 0
-      method: 2             # barrier
-      BarHomogeneous: 1     # Use homogeneous barrier if standard does not converge
-      BarConvTol: 1.e-5
-      FeasibilityTol: 1.e-5
-      OptimalityTol: 1.e-5
-      Seed: 123
-      threads: 8
-    cplex-default:
-      threads: 4
-      lpmethod: 4 # barrier
-      solutiontype: 2 # non basic solution, ie no crossover
-      barrier.convergetol: 1.e-5
-      feasopt.tolerance: 1.e-6
-    cbc-default: {} # Used in CI
-    glpk-default: {} # Used in CI
-
-  mem: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2
-
-
-plotting:
-  map:
-    boundaries: [-11, 30, 34, 71]
-    color_geomap:
-      ocean: white
-      land: white
-  eu_node_location:
-    x: -5.5
-    y: 46.
-  costs_max: 1000
-  costs_threshold: 1
-  energy_max: 20000
-  energy_min: -20000
-  energy_threshold: 50.
-  vre_techs:
-  - onwind
-  - offwind-ac
-  - offwind-dc
-  - solar
-  - ror
-  renewable_storage_techs:
-  - PHS
-  - hydro
-  conv_techs:
-  - OCGT
-  - CCGT
-  - Nuclear
-  - Coal
-  storage_techs:
-  - hydro+PHS
-  - battery
-  - H2
-  load_carriers:
-  - AC load
-  AC_carriers:
-  - AC line
-  - AC transformer
-  link_carriers:
-  - DC line
-  - Converter AC-DC
-  heat_links:
-  - heat pump
-  - resistive heater
-  - CHP heat
-  - CHP electric
-  - gas boiler
-  - central heat pump
-  - central resistive heater
-  - central CHP heat
-  - central CHP electric
-  - central gas boiler
-  heat_generators:
-  - gas boiler
-  - central gas boiler
-  - solar thermal collector
-  - central solar thermal collector
-
-  nice_names:
-    OCGT: "Open-Cycle Gas"
-    CCGT: "Combined-Cycle Gas"
-    offwind-ac: "Offshore Wind (AC)"
-    offwind-dc: "Offshore Wind (DC)"
-    onwind: "Onshore Wind"
-    solar: "Solar"
-    PHS: "Pumped Hydro Storage"
-    hydro: "Reservoir & Dam"
-    battery: "Battery Storage"
-    H2: "Hydrogen Storage"
-    lines: "Transmission Lines"
-    ror: "Run of River"
-
-  tech_colors:
-    # wind
-    onwind: "#235ebc"
-    onshore wind: "#235ebc"
-    offwind: "#6895dd"
-    offshore wind: "#6895dd"
-    offwind-ac: "#6895dd"
-    offshore wind (AC): "#6895dd"
-    offshore wind ac: "#6895dd"
-    offwind-dc: "#74c6f2"
-    offshore wind (DC): "#74c6f2"
-    offshore wind dc: "#74c6f2"
-    # water
-    hydro: '#298c81'
-    hydro reservoir: '#298c81'
-    ror: '#3dbfb0'
-    run of river: '#3dbfb0'
-    hydroelectricity: '#298c81'
-    PHS: '#51dbcc'
-    hydro+PHS: "#08ad97"
-    wave: '#a7d4cf'
-    # solar
-    solar: "#f9d002"
-    solar PV: "#f9d002"
-    solar thermal: '#ffbf2b'
-    solar rooftop: '#ffea80'
-    # gas
-    OCGT: '#e0986c'
-    OCGT marginal: '#e0986c'
-    OCGT-heat: '#e0986c'
-    gas boiler: '#db6a25'
-    gas boilers: '#db6a25'
-    gas boiler marginal: '#db6a25'
-    gas: '#e05b09'
-    fossil gas: '#e05b09'
-    natural gas: '#e05b09'
-    CCGT: '#a85522'
-    CCGT marginal: '#a85522'
-    allam: '#B98F76'
-    gas for industry co2 to atmosphere: '#692e0a'
-    gas for industry co2 to stored: '#8a3400'
-    gas for industry: '#853403'
-    gas for industry CC: '#692e0a'
-    gas pipeline: '#ebbca0'
-    gas pipeline new: '#a87c62'
-    # oil
-    oil: '#c9c9c9'
-    oil boiler: '#adadad'
-    agriculture machinery oil: '#949494'
-    shipping oil: "#808080"
-    land transport oil: '#afafaf'
-    # nuclear
-    Nuclear: '#ff8c00'
-    Nuclear marginal: '#ff8c00'
-    nuclear: '#ff8c00'
-    uranium: '#ff8c00'
-    # coal
-    Coal: '#545454'
-    coal: '#545454'
-    Coal marginal: '#545454'
-    solid: '#545454'
-    Lignite: '#826837'
-    lignite: '#826837'
-    Lignite marginal: '#826837'
-    # biomass
-    biogas: '#e3d37d'
-    biomass: '#baa741'
-    solid biomass: '#baa741'
-    solid biomass transport: '#baa741'
-    solid biomass for industry: '#7a6d26'
-    solid biomass for industry CC: '#47411c'
-    solid biomass for industry co2 from atmosphere: '#736412'
-    solid biomass for industry co2 to stored: '#47411c'
-    biomass boiler: '#8A9A5B'
-    biomass to liquid: '#32CD32'
-    BioSNG: '#123456'
-    # power transmission
-    lines: '#6c9459'
-    transmission lines: '#6c9459'
-    electricity distribution grid: '#97ad8c'
-    # electricity demand
-    Electric load: '#110d63'
-    electric demand: '#110d63'
-    electricity: '#110d63'
-    industry electricity: '#2d2a66'
-    industry new electricity: '#2d2a66'
-    agriculture electricity: '#494778'
-    # battery + EVs
-    battery: '#ace37f'
-    battery storage: '#ace37f'
-    home battery: '#80c944'
-    home battery storage: '#80c944'
-    BEV charger: '#baf238'
-    V2G: '#e5ffa8'
-    land transport EV: '#baf238'
-    Li ion: '#baf238'
-    # hot water storage
-    water tanks: '#e69487'
-    hot water storage: '#e69487'
-    hot water charging: '#e69487'
-    hot water discharging: '#e69487'
-    # heat demand
-    Heat load: '#cc1f1f'
-    heat: '#cc1f1f'
-    heat demand: '#cc1f1f'
-    rural heat: '#ff5c5c'
-    central heat: '#cc1f1f'
-    decentral heat: '#750606'
-    low-temperature heat for industry: '#8f2727'
-    process heat: '#ff0000'
-    agriculture heat: '#d9a5a5'
-    # heat supply
-    heat pumps: '#2fb537'
-    heat pump: '#2fb537'
-    air heat pump: '#36eb41'
-    ground heat pump: '#2fb537'
-    Ambient: '#98eb9d'
-    CHP: '#8a5751'
-    CHP CC: '#634643'
-    CHP heat: '#8a5751'
-    CHP electric: '#8a5751'
-    district heating: '#e8beac'
-    resistive heater: '#d8f9b8'
-    retrofitting: '#8487e8'
-    building retrofitting: '#8487e8'
-    # hydrogen
-    H2 for industry: "#f073da"
-    H2 for shipping: "#ebaee0"
-    H2: '#bf13a0'
-    hydrogen: '#bf13a0'
-    SMR: '#870c71'
-    SMR CC: '#4f1745'
-    H2 liquefaction: '#d647bd'
-    hydrogen storage: '#bf13a0'
-    H2 storage: '#bf13a0'
-    land transport fuel cell: '#6b3161'
-    H2 pipeline: '#f081dc'
-    H2 pipeline retrofitted: '#ba99b5'
-    H2 Fuel Cell: '#c251ae'
-    H2 Electrolysis: '#ff29d9'
-    # ammonia
-    NH3: '#46caf0'
-    ammonia: '#46caf0'
-    ammonia store: '#00ace0'
-    ammonia cracker: '#87d0e6'
-    Haber-Bosch: '#076987'
-    # syngas
-    Sabatier: '#9850ad'
-    methanation: '#c44ce6'
-    methane: '#c44ce6'
-    helmeth: '#e899ff'
-    # synfuels
-    Fischer-Tropsch: '#25c49a'
-    liquid: '#25c49a'
-    kerosene for aviation: '#a1ffe6'
-    naphtha for industry: '#57ebc4'
-    methanolisation: '#83d6d5'
-    methanol: '#468c8b'
-    shipping methanol: '#468c8b'
-    # co2
-    CC: '#f29dae'
-    CCS: '#f29dae'
-    CO2 sequestration: '#f29dae'
-    DAC: '#ff5270'
-    co2 stored: '#f2385a'
-    co2: '#f29dae'
-    co2 vent: '#ffd4dc'
-    CO2 pipeline: '#f5627f'
-    # emissions
-    process emissions CC: '#000000'
-    process emissions: '#222222'
-    process emissions to stored: '#444444'
-    process emissions to atmosphere: '#888888'
-    oil emissions: '#aaaaaa'
-    shipping oil emissions: "#555555"
-    shipping methanol emissions: '#666666'
-    land transport oil emissions: '#777777'
-    agriculture machinery oil emissions: '#333333'
-    # other
-    shipping: '#03a2ff'
-    power-to-heat: '#2fb537'
-    power-to-gas: '#c44ce6'
-    power-to-H2: '#ff29d9'
-    power-to-liquid: '#25c49a'
-    gas-to-power/heat: '#ee8340'
-    waste: '#e3d37d'
-    other: '#000000'
-    geothermal: '#ba91b1'
-    AC-AC: "#70af1d"
-    AC line: "#70af1d"
-    links: "#8a1caf"
-    HVDC links: "#8a1caf"
-    DC-DC: "#8a1caf"
-    DC link: "#8a1caf"

From d7d663b61b420fc3f564e0be2bbbd391c66b96d0 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Tue, 11 Apr 2023 02:14:04 +0200
Subject: [PATCH 007/176] Delete test directory

---
 test/config.electricity.yaml | 80 ------------------------------------
 test/config.myopic.yaml      | 79 -----------------------------------
 test/config.overnight.yaml   | 74 ---------------------------------
 3 files changed, 233 deletions(-)
 delete mode 100755 test/config.electricity.yaml
 delete mode 100644 test/config.myopic.yaml
 delete mode 100644 test/config.overnight.yaml

diff --git a/test/config.electricity.yaml b/test/config.electricity.yaml
deleted file mode 100755
index 6798e38c..00000000
--- a/test/config.electricity.yaml
+++ /dev/null
@@ -1,80 +0,0 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
-#
-# SPDX-License-Identifier: CC0-1.0
-
-tutorial: true
-
-
-run:
-  name: "test-elec" # use this to keep track of runs with different settings
-  disable_progressbar: true
-  shared_resources: true
-  shared_cutouts: true
-
-scenario:
-  clusters:
-  - 5
-  opts:
-  - Co2L-24H
-
-countries: ['BE']
-
-snapshots:
-  start: "2013-03-01"
-  end: "2013-03-08"
-
-electricity:
-  co2limit: 100.e+6
-
-  extendable_carriers:
-    Generator: [OCGT]
-    StorageUnit: [battery]
-    Store: [H2]
-    Link: [H2 pipeline]
-
-  renewable_carriers: [solar, onwind, offwind-ac, offwind-dc]
-
-
-atlite:
-  default_cutout: be-03-2013-era5
-  cutouts:
-    be-03-2013-era5:
-      module: era5
-      x: [4., 15.]
-      y: [46., 56.]
-      time: ["2013-03-01", "2013-03-08"]
-
-renewable:
-  onwind:
-    cutout: be-03-2013-era5
-  offwind-ac:
-    cutout: be-03-2013-era5
-    max_depth: false
-  offwind-dc:
-    cutout: be-03-2013-era5
-    max_depth: false
-  solar:
-    cutout: be-03-2013-era5
-
-
-clustering:
-  exclude_carriers: ["OCGT", "offwind-ac", "coal"]
-
-
-solving:
-  solver:
-    name: glpk
-    options: "glpk-default"
-
-
-plotting:
-  map:
-    boundaries:
-  eu_node_location:
-    x: -5.5
-    y: 46.
-  costs_max: 1000
-  costs_threshold: 0.0000001
-  energy_max:
-  energy_min:
-  energy_threshold: 0.000001
diff --git a/test/config.myopic.yaml b/test/config.myopic.yaml
deleted file mode 100644
index efa03136..00000000
--- a/test/config.myopic.yaml
+++ /dev/null
@@ -1,79 +0,0 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
-#
-# SPDX-License-Identifier: CC0-1.0
-
-tutorial: true
-
-run:
-  name: "test-sector-myopic"
-  disable_progressbar: true
-  shared_resources: true
-  shared_cutouts: true
-
-foresight: myopic
-
-scenario:
-  ll:
-  - v1.5
-  clusters:
-  - 5
-  sector_opts:
-  - 24H-T-H-B-I-A-solar+p3-dist1
-  planning_horizons:
-  - 2030
-  - 2040
-  - 2050
-
-countries: ['BE']
-
-snapshots:
-  start: "2013-03-01"
-  end: "2013-03-08"
-
-electricity:
-  renewable_carriers: [solar, onwind, offwind-ac, offwind-dc]
-
-atlite:
-  default_cutout: be-03-2013-era5
-  cutouts:
-    be-03-2013-era5:
-      module: era5
-      x: [4., 15.]
-      y: [46., 56.]
-      time: ["2013-03-01", "2013-03-08"]
-
-renewable:
-  onwind:
-    cutout: be-03-2013-era5
-  offwind-ac:
-    cutout: be-03-2013-era5
-    max_depth: false
-  offwind-dc:
-    cutout: be-03-2013-era5
-    max_depth: false
-  solar:
-    cutout: be-03-2013-era5
-
-industry:
-  St_primary_fraction:
-    2030: 0.6
-    2040: 0.5
-    2050: 0.4
-
-solving:
-  solver:
-    name: glpk
-    options: glpk-default
-  mem: 4000
-
-plotting:
-  map:
-    boundaries:
-  eu_node_location:
-    x: -5.5
-    y: 46.
-  costs_max: 1000
-  costs_threshold: 0.0000001
-  energy_max:
-  energy_min:
-  energy_threshold: 0.000001
diff --git a/test/config.overnight.yaml b/test/config.overnight.yaml
deleted file mode 100644
index fb468ded..00000000
--- a/test/config.overnight.yaml
+++ /dev/null
@@ -1,74 +0,0 @@
-# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
-#
-# SPDX-License-Identifier: CC0-1.0
-
-tutorial: true
-
-run:
-  name: "test-sector-overnight"
-  disable_progressbar: true
-  shared_resources: true
-  shared_cutouts: true
-
-
-scenario:
-  ll:
-  - v1.5
-  clusters:
-  - 5
-  sector_opts:
-  - CO2L0-24H-T-H-B-I-A-solar+p3-dist1
-  planning_horizons:
-  - 2030
-
-countries: ['BE']
-
-snapshots:
-  start: "2013-03-01"
-  end: "2013-03-08"
-
-electricity:
-  renewable_carriers: [solar, onwind, offwind-ac, offwind-dc]
-
-atlite:
-  default_cutout: be-03-2013-era5
-  cutouts:
-    be-03-2013-era5:
-      module: era5
-      x: [4., 15.]
-      y: [46., 56.]
-      time: ["2013-03-01", "2013-03-08"]
-
-renewable:
-  onwind:
-    cutout: be-03-2013-era5
-  offwind-ac:
-    cutout: be-03-2013-era5
-    max_depth: false
-  offwind-dc:
-    cutout: be-03-2013-era5
-    max_depth: false
-  solar:
-    cutout: be-03-2013-era5
-
-sector:
-  gas_network: true
-  H2_retrofit: true
-
-solving:
-  solver:
-    name: glpk
-    options: glpk-default
-  mem: 4000
-
-plotting:
-  map:
-    boundaries:
-  eu_node_location:
-    x: -5.5
-    y: 46.
-  costs_max: 1000
-  costs_threshold: 0.0000001
-  energy_max:
-  energy_min:
-  energy_threshold: 0.000001

From be1989fe2964025a55897c40cb2bc87baea75f77 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Tue, 11 Apr 2023 02:18:34 +0200
Subject: [PATCH 008/176] Update .syncignore-receive

---
 .syncignore-receive | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.syncignore-receive b/.syncignore-receive
index a216e0d8..d2e9b76d 100644
--- a/.syncignore-receive
+++ b/.syncignore-receive
@@ -18,3 +18,4 @@ cutouts
 data
 benchmarks
 *.nc
+configs

From d10ff9bdf1e4317e90b9151fa01e7749a1fb38ae Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Tue, 11 Apr 2023 02:25:21 +0200
Subject: [PATCH 009/176] Update Snakefile

---
 Snakefile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Snakefile b/Snakefile
index 621e4e9d..48529908 100644
--- a/Snakefile
+++ b/Snakefile
@@ -14,11 +14,11 @@ from snakemake.utils import min_version
 min_version("7.7")
 
 
-if not exists("config.yaml"):
-    copyfile("config.default.yaml", "config.yaml")
+if not exists("configs/config.yaml"):
+    copyfile("configs/config.default.yaml", "configs/config.yaml")
 
 
-configfile: "config.yaml"
+configfile: "configs/config.yaml"
 
 
 COSTS = f"data/costs_{config['costs']['year']}.csv"

From 9f641b4b2463f87d90aaed5f833d80d63f9fb9fd Mon Sep 17 00:00:00 2001
From: Ahmad <ahmedwahid16101@gmail.com>
Date: Tue, 11 Apr 2023 23:39:35 +0200
Subject: [PATCH 010/176] rename build_load_data to build_electricity_demand

Signed-off-by: Ahmad <ahmedwahid16101@gmail.com>
---
 rules/build_electricity.smk                                 | 6 +++---
 scripts/{build_load_data.py => build_electricity_demand.py} | 0
 2 files changed, 3 insertions(+), 3 deletions(-)
 rename scripts/{build_load_data.py => build_electricity_demand.py} (100%)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index 9be7408d..0074a8a9 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -18,19 +18,19 @@ if config["enable"].get("prepare_links_p_nom", False):
             "../scripts/prepare_links_p_nom.py"
 
 
-rule build_load_data:
+rule build_electricity_demand:
     input:
         ancient("data/load_raw.csv"),
     output:
         RESOURCES + "load.csv",
     log:
-        LOGS + "build_load_data.log",
+        LOGS + "build_electricity_demand.log",
     resources:
         mem_mb=5000,
     conda:
         "../envs/environment.yaml"
     script:
-        "../scripts/build_load_data.py"
+        "../scripts/build_electricity_demand.py"
 
 
 rule build_powerplants:
diff --git a/scripts/build_load_data.py b/scripts/build_electricity_demand.py
similarity index 100%
rename from scripts/build_load_data.py
rename to scripts/build_electricity_demand.py

From f3872c943306f8a4b88b56c6968962334ee44a0b Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Wed, 12 Apr 2023 02:45:28 +0200
Subject: [PATCH 011/176] Update copy_config.py

insert the new path for config for the copying process, solve the error in rule copy_config
---
 scripts/copy_config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/copy_config.py b/scripts/copy_config.py
index c79e578c..06531f03 100644
--- a/scripts/copy_config.py
+++ b/scripts/copy_config.py
@@ -13,7 +13,7 @@ from shutil import copy
 import yaml
 
 files = {
-    "config.yaml": "config.yaml",
+    "configs/config.yaml": "config.yaml",
     "Snakefile": "Snakefile",
     "scripts/solve_network.py": "solve_network.py",
     "scripts/prepare_sector_network.py": "prepare_sector_network.py",

From 55315e30c019cde95a82620b6aba68ae88ebeeef Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Thu, 13 Apr 2023 04:06:23 +0200
Subject: [PATCH 012/176] Update ci.yaml

---
 .github/workflows/ci.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index e9fedd36..2e7de2d3 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -103,9 +103,9 @@ jobs:
       run: |
         conda activate pypsa-eur
         conda list
-        snakemake -call solve_elec_networks --configfile test/config.electricity.yaml --rerun-triggers=mtime
-        snakemake -call all --configfile test/config.overnight.yaml --rerun-triggers=mtime
-        snakemake -call all --configfile test/config.myopic.yaml --rerun-triggers=mtime
+        snakemake -call solve_elec_networks --configfile configs/test/config.electricity.yaml --rerun-triggers=mtime
+        snakemake -call all --configfile configs/test/config.overnight.yaml --rerun-triggers=mtime
+        snakemake -call all --configfile configs/test/config.myopic.yaml --rerun-triggers=mtime
 
     - name: Upload artifacts
       uses: actions/upload-artifact@v3

From 90d8f0ac3f303b9b39455aadafe6dd7602d36944 Mon Sep 17 00:00:00 2001
From: Ahmad <ahmedwahid16101@gmail.com>
Date: Tue, 11 Apr 2023 23:39:35 +0200
Subject: [PATCH 013/176] rename build_load_data to build_electricity_demand

Signed-off-by: Ahmad <ahmedwahid16101@gmail.com>
---
 rules/build_electricity.smk                                 | 6 +++---
 scripts/{build_load_data.py => build_electricity_demand.py} | 0
 2 files changed, 3 insertions(+), 3 deletions(-)
 rename scripts/{build_load_data.py => build_electricity_demand.py} (100%)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index 9be7408d..0074a8a9 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -18,19 +18,19 @@ if config["enable"].get("prepare_links_p_nom", False):
             "../scripts/prepare_links_p_nom.py"
 
 
-rule build_load_data:
+rule build_electricity_demand:
     input:
         ancient("data/load_raw.csv"),
     output:
         RESOURCES + "load.csv",
     log:
-        LOGS + "build_load_data.log",
+        LOGS + "build_electricity_demand.log",
     resources:
         mem_mb=5000,
     conda:
         "../envs/environment.yaml"
     script:
-        "../scripts/build_load_data.py"
+        "../scripts/build_electricity_demand.py"
 
 
 rule build_powerplants:
diff --git a/scripts/build_load_data.py b/scripts/build_electricity_demand.py
similarity index 100%
rename from scripts/build_load_data.py
rename to scripts/build_electricity_demand.py

From 763d77d19d0614eed198c23bc97338eebf61995c Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 14 Apr 2023 19:05:42 +0000
Subject: [PATCH 014/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/build_cop_profiles.py | 1 +
 scripts/build_gas_network.py  | 1 +
 2 files changed, 2 insertions(+)

diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py
index 94cc8fb4..91d03469 100644
--- a/scripts/build_cop_profiles.py
+++ b/scripts/build_cop_profiles.py
@@ -12,6 +12,7 @@ sink.
 
 The quadratic regression used is based on Staffell et al. (2012)
 
+
 https://doi.org/10.1039/C2EE22653G.
 """
 
diff --git a/scripts/build_gas_network.py b/scripts/build_gas_network.py
index 907b151c..a9f28e0d 100644
--- a/scripts/build_gas_network.py
+++ b/scripts/build_gas_network.py
@@ -29,6 +29,7 @@ def diameter_to_capacity(pipe_diameter_mm):
 
     Based on p.15 of
 
+
     https://gasforclimate2050.eu/wp-content/uploads/2020/07/2020_European-Hydrogen-Backbone_Report.pdf
     """
     # slopes definitions

From 2984b3bb993fec591b48ce27c767d1fb3bfb237a Mon Sep 17 00:00:00 2001
From: Ahmad <ahmedwahid16101@gmail.com>
Date: Fri, 14 Apr 2023 21:26:25 +0200
Subject: [PATCH 015/176] updated docs from build_load_data to
 build_electricity_demand

Signed-off-by: Ahmad <ahmedwahid16101@gmail.com>
---
 doc/tutorial.rst        | 4 ++--
 doc/tutorial_sector.rst | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/tutorial.rst b/doc/tutorial.rst
index b6f5311b..82ccc55a 100644
--- a/doc/tutorial.rst
+++ b/doc/tutorial.rst
@@ -155,7 +155,7 @@ This triggers a workflow of multiple preceding jobs that depend on each rule's i
         19[label = "build_hydro_profile", color = "0.44 0.6 0.85", style="rounded"];
         20[label = "retrieve_cost_data", color = "0.30 0.6 0.85", style="rounded"];
         21[label = "build_powerplants", color = "0.16 0.6 0.85", style="rounded"];
-        22[label = "build_load_data", color = "0.00 0.6 0.85", style="rounded"];
+        22[label = "build_electricity_demand", color = "0.00 0.6 0.85", style="rounded"];
         23[label = "retrieve_load_data", color = "0.34 0.6 0.85", style="rounded,dashed"];
         1 -> 0
         2 -> 1
@@ -232,7 +232,7 @@ In the terminal, this will show up as a list of jobs to be run:
     base_network                    1              1              1
     build_bus_regions               1              1              1
     build_hydro_profile             1              1              1
-    build_load_data                 1              1              1
+    build_electricity_demand        1              1              1
     build_powerplants               1              1              1
     build_renewable_profiles        4              1              1
     build_shapes                    1              1              1
diff --git a/doc/tutorial_sector.rst b/doc/tutorial_sector.rst
index 1d8e8ba5..1cbd6779 100644
--- a/doc/tutorial_sector.rst
+++ b/doc/tutorial_sector.rst
@@ -140,7 +140,7 @@ successfully.
         18[label = "retrieve_ship_raster", color = "0.15 0.6 0.85", style="rounded"];
         19[label = "retrieve_cost_data", color = "0.50 0.6 0.85", style="rounded"];
         20[label = "build_powerplants", color = "0.49 0.6 0.85", style="rounded"];
-        21[label = "build_load_data", color = "0.39 0.6 0.85", style="rounded"];
+        21[label = "build_electricity_demand", color = "0.39 0.6 0.85", style="rounded"];
         22[label = "retrieve_load_data", color = "0.05 0.6 0.85", style="rounded"];
         23[label = "build_gas_input_locations", color = "0.45 0.6 0.85", style="rounded"];
         24[label = "prepare_network", color = "0.31 0.6 0.85", style="rounded"];
@@ -367,7 +367,7 @@ implemented in the workflow:
         18[label = "retrieve_ship_raster", color = "0.09 0.6 0.85", style="rounded"];
         19[label = "retrieve_cost_data", color = "0.04 0.6 0.85", style="rounded"];
         20[label = "build_powerplants", color = "0.28 0.6 0.85", style="rounded"];
-        21[label = "build_load_data", color = "0.46 0.6 0.85", style="rounded"];
+        21[label = "build_electricity_demand", color = "0.46 0.6 0.85", style="rounded"];
         22[label = "retrieve_load_data", color = "0.44 0.6 0.85", style="rounded"];
         23[label = "build_energy_totals", color = "0.53 0.6 0.85", style="rounded"];
         24[label = "build_population_weighted_energy_totals", color = "0.03 0.6 0.85", style="rounded"];

From b877b41019ffa4ed8da0ee703a0b875b246c89d4 Mon Sep 17 00:00:00 2001
From: Ahmad <ahmedwahid16101@gmail.com>
Date: Sun, 16 Apr 2023 02:28:56 +0200
Subject: [PATCH 016/176] - incorporated the remaining changes - added a
 released note about the renaming of build_load_data to
 build_electricity_demand

---
 doc/preparation.rst                 | 6 +++---
 doc/release_notes.rst               | 4 +++-
 scripts/build_electricity_demand.py | 2 +-
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/doc/preparation.rst b/doc/preparation.rst
index a180bb0d..b5a12062 100644
--- a/doc/preparation.rst
+++ b/doc/preparation.rst
@@ -86,13 +86,13 @@ Rule ``build_powerplants``
 .. automodule:: build_powerplants
 
 
-.. _load_data:
+.. _electricity_demand:
 
-Rule ``build_load_data``
+Rule ``build_electricity_demand``
 =============================
 
 
-.. automodule:: build_load_data
+.. automodule:: build_electricity_demand
 
 .. _ship:
 
diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index e668e92e..3d05f27c 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -12,6 +12,8 @@ Upcoming Release
 
 * Bugfix: Correct typo in the CPLEX solver configuration in ``config.default.yaml``.
 
+* Renamed script file from PyPSA-EUR ``build_load_data`` to ``build_electricity_demand``.
+
 PyPSA-Eur 0.8.0 (18th March 2023)
 =================================
 
@@ -558,7 +560,7 @@ More OPSD integration:
   This will overwrite the capacities calculated from the heuristic approach in :func:`estimate_renewable_capacities()`
   [`#212 <https://github.com/PyPSA/pypsa-eur/pull/212>`_].
 
-* Electricity consumption data is now retrieved directly from the `OPSD website <https://data.open-power-system-data.org/time_series/2019-06-05>`_ using the rule :mod:`build_load_data`.
+* Electricity consumption data is now retrieved directly from the `OPSD website <https://data.open-power-system-data.org/time_series/2019-06-05>`_ using the rule :mod:`build_electricity_demand`.
   The user can decide whether to take the ENTSO-E power statistics data (default) or the ENTSO-E transparency data
   [`#211 <https://github.com/PyPSA/pypsa-eur/pull/211>`_].
 
diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py
index 01e3fb1e..4bf0306b 100755
--- a/scripts/build_electricity_demand.py
+++ b/scripts/build_electricity_demand.py
@@ -276,7 +276,7 @@ if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
 
-        snakemake = mock_snakemake("build_load_data")
+        snakemake = mock_snakemake("build_electricity_demand")
 
     configure_logging(snakemake)
 

From 05b0a81808c63de36cc90fef489225160522293e Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sun, 16 Apr 2023 13:32:22 +0000
Subject: [PATCH 017/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/build_cop_profiles.py | 1 +
 scripts/build_gas_network.py  | 1 +
 2 files changed, 2 insertions(+)

diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py
index 94cc8fb4..91d03469 100644
--- a/scripts/build_cop_profiles.py
+++ b/scripts/build_cop_profiles.py
@@ -12,6 +12,7 @@ sink.
 
 The quadratic regression used is based on Staffell et al. (2012)
 
+
 https://doi.org/10.1039/C2EE22653G.
 """
 
diff --git a/scripts/build_gas_network.py b/scripts/build_gas_network.py
index 907b151c..a9f28e0d 100644
--- a/scripts/build_gas_network.py
+++ b/scripts/build_gas_network.py
@@ -29,6 +29,7 @@ def diameter_to_capacity(pipe_diameter_mm):
 
     Based on p.15 of
 
+
     https://gasforclimate2050.eu/wp-content/uploads/2020/07/2020_European-Hydrogen-Backbone_Report.pdf
     """
     # slopes definitions

From 27dc9cedfdbe4aa716552d212eb2749f77ae0a00 Mon Sep 17 00:00:00 2001
From: lisazeyen <lisa.zeyen@web.de>
Date: Thu, 20 Apr 2023 09:35:53 +0200
Subject: [PATCH 018/176] fix bugs in reserve constraint

---
 scripts/solve_network.py | 30 +++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index cfb95bfe..c08c7b09 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -43,6 +43,7 @@ from vresutils.benchmark import memory_logger
 
 logger = logging.getLogger(__name__)
 pypsa.pf.logger.setLevel(logging.WARNING)
+from pypsa.descriptors import get_switchable_as_dense as get_as_dense
 
 
 def add_land_use_constraint(n, config):
@@ -414,7 +415,7 @@ def add_operational_reserve_margin(n, sns, config):
         0, np.inf, coords=[sns, n.generators.index], name="Generator-r"
     )
     reserve = n.model["Generator-r"]
-    lhs = reserve.sum("Generator")
+    summed_reserve = reserve.sum("Generator")
 
     # Share of extendable renewable capacities
     ext_i = n.generators.query("p_nom_extendable").index
@@ -426,10 +427,10 @@ def add_operational_reserve_margin(n, sns, config):
             .loc[vres_i.intersection(ext_i)]
             .rename({"Generator-ext": "Generator"})
         )
-        lhs = lhs + (p_nom_vres * (-EPSILON_VRES * capacity_factor)).sum()
+        lhs = summed_reserve + (p_nom_vres * (-EPSILON_VRES * capacity_factor)).sum("Generator")
 
     # Total demand per t
-    demand = n.loads_t.p_set.sum(axis=1)
+    demand = get_as_dense(n, "Load", "p_set").sum(axis=1)
 
     # VRES potential of non extendable generators
     capacity_factor = n.generators_t.p_max_pu[vres_i.difference(ext_i)]
@@ -441,17 +442,24 @@ def add_operational_reserve_margin(n, sns, config):
 
     n.model.add_constraints(lhs >= rhs, name="reserve_margin")
 
+    # additional contraint that capacity is not exceeded
+    gen_i = n.generators.index
+    ext_i = n.generators.query('p_nom_extendable').index
+    fix_i = n.generators.query('not p_nom_extendable').index
+
+    dispatch = n.model["Generator-p"]
     reserve = n.model["Generator-r"]
 
-    lhs = n.model.constraints["Generator-fix-p-upper"].lhs
-    lhs = lhs + reserve.loc[:, lhs.coords["Generator-fix"]].drop("Generator")
-    rhs = n.model.constraints["Generator-fix-p-upper"].rhs
-    n.model.add_constraints(lhs <= rhs, name="Generator-fix-p-upper-reserve")
+    capacity_variable = n.model["Generator-p_nom"].rename({"Generator-ext": "Generator"})
+    capacity_fixed = n.generators.p_nom[fix_i]
 
-    lhs = n.model.constraints["Generator-ext-p-upper"].lhs
-    lhs = lhs + reserve.loc[:, lhs.coords["Generator-ext"]].drop("Generator")
-    rhs = n.model.constraints["Generator-ext-p-upper"].rhs
-    n.model.add_constraints(lhs >= rhs, name="Generator-ext-p-upper-reserve")
+    p_max_pu = get_as_dense(n, 'Generator', 'p_max_pu')
+
+    lhs = dispatch + reserve - capacity_variable * p_max_pu[ext_i]
+
+    rhs = (p_max_pu[fix_i] * capacity_fixed).reindex(columns=gen_i, fill_value=0)
+
+    n.model.add_constraints(lhs <= rhs, name="updated_capacity_constraint")
 
 
 def add_battery_constraints(n):

From 51785524a3e28db65d6444777a706dcf144efc1c Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 20 Apr 2023 07:39:15 +0000
Subject: [PATCH 019/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/solve_network.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index c08c7b09..30540da0 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -427,7 +427,9 @@ def add_operational_reserve_margin(n, sns, config):
             .loc[vres_i.intersection(ext_i)]
             .rename({"Generator-ext": "Generator"})
         )
-        lhs = summed_reserve + (p_nom_vres * (-EPSILON_VRES * capacity_factor)).sum("Generator")
+        lhs = summed_reserve + (p_nom_vres * (-EPSILON_VRES * capacity_factor)).sum(
+            "Generator"
+        )
 
     # Total demand per t
     demand = get_as_dense(n, "Load", "p_set").sum(axis=1)
@@ -444,16 +446,18 @@ def add_operational_reserve_margin(n, sns, config):
 
     # additional contraint that capacity is not exceeded
     gen_i = n.generators.index
-    ext_i = n.generators.query('p_nom_extendable').index
-    fix_i = n.generators.query('not p_nom_extendable').index
+    ext_i = n.generators.query("p_nom_extendable").index
+    fix_i = n.generators.query("not p_nom_extendable").index
 
     dispatch = n.model["Generator-p"]
     reserve = n.model["Generator-r"]
 
-    capacity_variable = n.model["Generator-p_nom"].rename({"Generator-ext": "Generator"})
+    capacity_variable = n.model["Generator-p_nom"].rename(
+        {"Generator-ext": "Generator"}
+    )
     capacity_fixed = n.generators.p_nom[fix_i]
 
-    p_max_pu = get_as_dense(n, 'Generator', 'p_max_pu')
+    p_max_pu = get_as_dense(n, "Generator", "p_max_pu")
 
     lhs = dispatch + reserve - capacity_variable * p_max_pu[ext_i]
 

From 1cd658c03fe3168154604cad7f8fcf07909c455b Mon Sep 17 00:00:00 2001
From: lisazeyen <35347358+lisazeyen@users.noreply.github.com>
Date: Thu, 20 Apr 2023 09:55:50 +0200
Subject: [PATCH 020/176] rename constraint

rename constraint to fit to other syntax

Co-authored-by: Fabian Hofmann <fab.hof@gmx.de>
---
 scripts/solve_network.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index 30540da0..5b580bac 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -463,7 +463,7 @@ def add_operational_reserve_margin(n, sns, config):
 
     rhs = (p_max_pu[fix_i] * capacity_fixed).reindex(columns=gen_i, fill_value=0)
 
-    n.model.add_constraints(lhs <= rhs, name="updated_capacity_constraint")
+    n.model.add_constraints(lhs <= rhs, name="Generator-p-reserve-upper")
 
 
 def add_battery_constraints(n):

From cd28fbf857af8a41bb151bce06eb0a9d9a8bf5e9 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Thu, 20 Apr 2023 10:56:41 +0200
Subject: [PATCH 021/176] Deactivate helmeth by default

---
 config.default.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config.default.yaml b/config.default.yaml
index 5218bc22..1305456a 100755
--- a/config.default.yaml
+++ b/config.default.yaml
@@ -464,7 +464,7 @@ sector:
   solar_cf_correction: 0.788457  # =  >>> 1/1.2683
   marginal_cost_storage: 0. #1e-4
   methanation: true
-  helmeth: true
+  helmeth: false
   coal_cc: false
   dac: true
   co2_vent: false

From 6d7eae8ad28169f73f399197bc46eaad85e15c62 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Thu, 20 Apr 2023 19:56:23 +0200
Subject: [PATCH 022/176] build_ship_raster: save reduced ship raster as tif
 file

---
 rules/build_electricity.smk  | 4 ++--
 scripts/build_ship_raster.py | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index 0074a8a9..133ceb6e 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -172,7 +172,7 @@ rule build_ship_raster:
             ],
         ),
     output:
-        RESOURCES + "shipdensity_raster.nc",
+        RESOURCES + "shipdensity_raster.tif",
     log:
         LOGS + "build_ship_raster.log",
     resources:
@@ -202,7 +202,7 @@ rule build_renewable_profiles:
             )
         ),
         ship_density=lambda w: (
-            RESOURCES + "shipdensity_raster.nc"
+            RESOURCES + "shipdensity_raster.tif"
             if "ship_threshold" in config["renewable"][w.technology].keys()
             else []
         ),
diff --git a/scripts/build_ship_raster.py b/scripts/build_ship_raster.py
index 59610ade..9d9dda0b 100644
--- a/scripts/build_ship_raster.py
+++ b/scripts/build_ship_raster.py
@@ -46,7 +46,7 @@ import logging
 import os
 import zipfile
 
-import rioxarray as rio
+import rioxarray
 from _helpers import configure_logging
 from build_natura_raster import determine_cutout_xXyY
 
@@ -64,10 +64,10 @@ if __name__ == "__main__":
 
     with zipfile.ZipFile(snakemake.input.ship_density) as zip_f:
         zip_f.extract("shipdensity_global.tif")
-        with rio.open_rasterio("shipdensity_global.tif") as ship_density:
+        with rioxarray.open_rasterio("shipdensity_global.tif") as ship_density:
             ship_density = ship_density.drop(["band"]).sel(
                 x=slice(min(xs), max(Xs)), y=slice(max(Ys), min(ys))
             )
-            ship_density.to_netcdf(snakemake.output[0])
+            ship_density.rio.to_raster(snakemake.output[0])
 
     os.remove("shipdensity_global.tif")

From 79501b5741479e61f8ff740b166863b28c12c162 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Thu, 20 Apr 2023 20:37:27 +0200
Subject: [PATCH 023/176] build_biomass: fix geopandas compat with pandas v2.0

---
 scripts/build_biomass_potentials.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py
index 50f2583c..21d0e623 100644
--- a/scripts/build_biomass_potentials.py
+++ b/scripts/build_biomass_potentials.py
@@ -197,7 +197,7 @@ def convert_nuts2_to_regions(bio_nuts2, regions):
     )
     overlay[adjust_cols] = overlay[adjust_cols].multiply(overlay["share"], axis=0)
 
-    bio_regions = overlay.groupby("name").sum()
+    bio_regions = overlay.dissolve("name", aggfunc="sum")
 
     bio_regions.drop(["area_nuts2", "share"], axis=1, inplace=True)
 

From 5c8e0080c8c95102dbdc232ec76831d5c2219317 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Fri, 21 Apr 2023 10:11:44 +0200
Subject: [PATCH 024/176] pre-commit: update docformatter version

---
 .pre-commit-config.yaml                                     | 2 +-
 scripts/add_brownfield.py                                   | 1 -
 scripts/add_existing_baseyear.py                            | 1 -
 scripts/build_biomass_transport_costs.py                    | 1 -
 scripts/build_bus_regions.py                                | 1 -
 scripts/build_clustered_population_layouts.py               | 1 -
 scripts/build_cop_profiles.py                               | 1 -
 scripts/build_cutout.py                                     | 1 -
 scripts/build_electricity_demand.py                         | 1 -
 scripts/build_energy_totals.py                              | 1 -
 scripts/build_gas_input_locations.py                        | 1 -
 scripts/build_gas_network.py                                | 1 -
 scripts/build_heat_demand.py                                | 1 -
 scripts/build_hydro_profile.py                              | 1 -
 scripts/build_industrial_distribution_key.py                | 1 -
 scripts/build_industrial_energy_demand_per_country_today.py | 1 -
 scripts/build_industrial_energy_demand_per_node.py          | 1 -
 scripts/build_industrial_energy_demand_per_node_today.py    | 1 -
 scripts/build_industrial_production_per_country.py          | 1 -
 scripts/build_industrial_production_per_country_tomorrow.py | 1 -
 scripts/build_industrial_production_per_node.py             | 1 -
 scripts/build_industry_sector_ratios.py                     | 1 -
 scripts/build_natura_raster.py                              | 1 -
 scripts/build_population_layouts.py                         | 1 -
 scripts/build_population_weighted_energy_totals.py          | 1 -
 scripts/build_renewable_profiles.py                         | 1 -
 scripts/build_retro_cost.py                                 | 1 -
 scripts/build_salt_cavern_potentials.py                     | 1 -
 scripts/build_sequestration_potentials.py                   | 1 -
 scripts/build_shapes.py                                     | 1 -
 scripts/build_ship_raster.py                                | 1 -
 scripts/build_solar_thermal_profiles.py                     | 1 -
 scripts/build_temperature_profiles.py                       | 1 -
 scripts/build_transport_demand.py                           | 1 -
 scripts/cluster_gas_network.py                              | 1 -
 scripts/copy_config.py                                      | 1 -
 scripts/make_summary.py                                     | 1 -
 scripts/plot_network.py                                     | 1 -
 scripts/plot_summary.py                                     | 1 -
 scripts/prepare_links_p_nom.py                              | 1 -
 scripts/prepare_sector_network.py                           | 1 -
 scripts/retrieve_databundle.py                              | 1 -
 scripts/solve_network.py                                    | 3 +--
 scripts/solve_operations_network.py                         | 1 -
 44 files changed, 2 insertions(+), 45 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 7e2f1149..4279b96a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -39,7 +39,7 @@ repos:
 
   # Make docstrings PEP 257 compliant
 - repo: https://github.com/PyCQA/docformatter
-  rev: v1.6.0
+  rev: v1.6.1-rc1
   hooks:
   - id: docformatter
     args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"]
diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py
index dc782838..e08b86d7 100644
--- a/scripts/add_brownfield.py
+++ b/scripts/add_brownfield.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Prepares brownfield data from previous planning horizon.
 """
diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py
index 992f3f98..5bc0960d 100644
--- a/scripts/add_existing_baseyear.py
+++ b/scripts/add_existing_baseyear.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Adds existing power and heat generation capacities for initial planning
 horizon.
diff --git a/scripts/build_biomass_transport_costs.py b/scripts/build_biomass_transport_costs.py
index 8b6b6927..9271b600 100644
--- a/scripts/build_biomass_transport_costs.py
+++ b/scripts/build_biomass_transport_costs.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Reads biomass transport costs for different countries of the JRC report.
 
diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py
index f1a0558a..cfbf90f7 100644
--- a/scripts/build_bus_regions.py
+++ b/scripts/build_bus_regions.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Creates Voronoi shapes for each bus representing both onshore and offshore
 regions.
diff --git a/scripts/build_clustered_population_layouts.py b/scripts/build_clustered_population_layouts.py
index bc81c29a..083f3de4 100644
--- a/scripts/build_clustered_population_layouts.py
+++ b/scripts/build_clustered_population_layouts.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Build population layouts for all clustered model regions as total as well as
 split by urban and rural population.
diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py
index 91d03469..342bd7e0 100644
--- a/scripts/build_cop_profiles.py
+++ b/scripts/build_cop_profiles.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Build coefficient of performance (COP) time series for air- or ground-sourced
 heat pumps.
diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py
index fd148cac..eefa881b 100644
--- a/scripts/build_cutout.py
+++ b/scripts/build_cutout.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Create cutouts with `atlite <https://atlite.readthedocs.io/en/latest/>`_.
 
diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py
index 4bf0306b..30f24f5c 100755
--- a/scripts/build_electricity_demand.py
+++ b/scripts/build_electricity_demand.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020 @JanFrederickUnnewehr, The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 This rule downloads the load data from `Open Power System Data Time series.
 
diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py
index f11c134f..45fc960f 100644
--- a/scripts/build_energy_totals.py
+++ b/scripts/build_energy_totals.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Build total energy demands per country using JRC IDEES, eurostat, and EEA data.
 """
diff --git a/scripts/build_gas_input_locations.py b/scripts/build_gas_input_locations.py
index 314cb767..a3b945ab 100644
--- a/scripts/build_gas_input_locations.py
+++ b/scripts/build_gas_input_locations.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Build import locations for fossil gas from entry-points, LNG terminals and
 production sites with data from SciGRID_gas and Global Energy Monitor.
diff --git a/scripts/build_gas_network.py b/scripts/build_gas_network.py
index a9f28e0d..f3021022 100644
--- a/scripts/build_gas_network.py
+++ b/scripts/build_gas_network.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Preprocess gas network based on data from bthe SciGRID_gas project
 (https://www.gas.scigrid.de/).
diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py
index 6164d25d..56ceb4b7 100644
--- a/scripts/build_heat_demand.py
+++ b/scripts/build_heat_demand.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Build heat demand time series using heating degree day (HDD) approximation.
 """
diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py
index 279bd7a1..9adf7112 100644
--- a/scripts/build_hydro_profile.py
+++ b/scripts/build_hydro_profile.py
@@ -4,7 +4,6 @@
 # SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Build hydroelectric inflow time-series for each country.
 
diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py
index d4518d1c..69daf64d 100644
--- a/scripts/build_industrial_distribution_key.py
+++ b/scripts/build_industrial_distribution_key.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Build spatial distribution of industries from Hotmaps database.
 """
diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py
index 3fa9ef56..703997b1 100644
--- a/scripts/build_industrial_energy_demand_per_country_today.py
+++ b/scripts/build_industrial_energy_demand_per_country_today.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Build industrial energy demand per country.
 """
diff --git a/scripts/build_industrial_energy_demand_per_node.py b/scripts/build_industrial_energy_demand_per_node.py
index b75b2058..55c10c5d 100644
--- a/scripts/build_industrial_energy_demand_per_node.py
+++ b/scripts/build_industrial_energy_demand_per_node.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Build industrial energy demand per model region.
 """
diff --git a/scripts/build_industrial_energy_demand_per_node_today.py b/scripts/build_industrial_energy_demand_per_node_today.py
index da1ec2d7..d845e704 100644
--- a/scripts/build_industrial_energy_demand_per_node_today.py
+++ b/scripts/build_industrial_energy_demand_per_node_today.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Build industrial energy demand per model region.
 """
diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py
index 1c2e4557..437806b3 100644
--- a/scripts/build_industrial_production_per_country.py
+++ b/scripts/build_industrial_production_per_country.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Build industrial production per country.
 """
diff --git a/scripts/build_industrial_production_per_country_tomorrow.py b/scripts/build_industrial_production_per_country_tomorrow.py
index e554e159..6c445608 100644
--- a/scripts/build_industrial_production_per_country_tomorrow.py
+++ b/scripts/build_industrial_production_per_country_tomorrow.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Build future industrial production per country.
 """
diff --git a/scripts/build_industrial_production_per_node.py b/scripts/build_industrial_production_per_node.py
index 7b40cb7c..7b69948a 100644
--- a/scripts/build_industrial_production_per_node.py
+++ b/scripts/build_industrial_production_per_node.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Build industrial production per model region.
 """
diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py
index 5f5f7b90..54f2cfdc 100644
--- a/scripts/build_industry_sector_ratios.py
+++ b/scripts/build_industry_sector_ratios.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Build specific energy consumption by carrier and industries.
 """
diff --git a/scripts/build_natura_raster.py b/scripts/build_natura_raster.py
index 9246508e..12011cd0 100644
--- a/scripts/build_natura_raster.py
+++ b/scripts/build_natura_raster.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Rasters the vector data of the `Natura 2000.
 
diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py
index a9308b7e..e864d925 100644
--- a/scripts/build_population_layouts.py
+++ b/scripts/build_population_layouts.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Build mapping between cutout grid cells and population (total, urban, rural).
 """
diff --git a/scripts/build_population_weighted_energy_totals.py b/scripts/build_population_weighted_energy_totals.py
index 85d7a8b3..879e3b9b 100644
--- a/scripts/build_population_weighted_energy_totals.py
+++ b/scripts/build_population_weighted_energy_totals.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Distribute country-level energy demands by population.
 """
diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py
index ea4dfa9a..6f03d746 100644
--- a/scripts/build_renewable_profiles.py
+++ b/scripts/build_renewable_profiles.py
@@ -4,7 +4,6 @@
 # SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Calculates for each network node the (i) installable capacity (based on land-
 use), (ii) the available generation time series (based on weather data), and
diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py
index 72792c92..cd1dae8d 100644
--- a/scripts/build_retro_cost.py
+++ b/scripts/build_retro_cost.py
@@ -3,7 +3,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 This script calculates the space heating savings through better insulation of
 the thermal envelope of a building and corresponding costs for different
diff --git a/scripts/build_salt_cavern_potentials.py b/scripts/build_salt_cavern_potentials.py
index 3fed7f91..956ed431 100644
--- a/scripts/build_salt_cavern_potentials.py
+++ b/scripts/build_salt_cavern_potentials.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Build salt cavern potentials for hydrogen storage.
 
diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py
index e17d4f0e..012effe8 100644
--- a/scripts/build_sequestration_potentials.py
+++ b/scripts/build_sequestration_potentials.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Build regionalised geological sequestration potential for carbon dioxide using
 data from `CO2Stop <https://setis.ec.europa.eu/european-co2-storage-
diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py
index 93ff0523..fda7613f 100644
--- a/scripts/build_shapes.py
+++ b/scripts/build_shapes.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Creates GIS shape files of the countries, exclusive economic zones and `NUTS3 <
 https://en.wikipedia.org/wiki/Nomenclature_of_Territorial_Units_for_Statistics>
diff --git a/scripts/build_ship_raster.py b/scripts/build_ship_raster.py
index 9d9dda0b..abfcdfa1 100644
--- a/scripts/build_ship_raster.py
+++ b/scripts/build_ship_raster.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2022 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Transforms the global ship density data from the `World Bank Data Catalogue.
 
diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py
index 1cec57f1..f4eb1557 100644
--- a/scripts/build_solar_thermal_profiles.py
+++ b/scripts/build_solar_thermal_profiles.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Build solar thermal collector time series.
 """
diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py
index b1973fd8..8f6d6c6c 100644
--- a/scripts/build_temperature_profiles.py
+++ b/scripts/build_temperature_profiles.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Build time series for air and soil temperatures per clustered model region.
 """
diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py
index 29973da3..6b8bd04f 100644
--- a/scripts/build_transport_demand.py
+++ b/scripts/build_transport_demand.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Build land transport demand per clustered model region including efficiency
 improvements due to drivetrain changes, time series for electric vehicle
diff --git a/scripts/cluster_gas_network.py b/scripts/cluster_gas_network.py
index 44188638..e7554dff 100755
--- a/scripts/cluster_gas_network.py
+++ b/scripts/cluster_gas_network.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Cluster gas transmission network to clustered model regions.
 """
diff --git a/scripts/copy_config.py b/scripts/copy_config.py
index c79e578c..d2108f4d 100644
--- a/scripts/copy_config.py
+++ b/scripts/copy_config.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Copy used configuration files and important scripts for archiving.
 """
diff --git a/scripts/make_summary.py b/scripts/make_summary.py
index 7b33aed8..3d743942 100644
--- a/scripts/make_summary.py
+++ b/scripts/make_summary.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Create summary CSV files for all scenario runs including costs, capacities,
 capacity factors, curtailment, energy balances, prices and other metrics.
diff --git a/scripts/plot_network.py b/scripts/plot_network.py
index 7c194b0c..0a22b2e5 100644
--- a/scripts/plot_network.py
+++ b/scripts/plot_network.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Creates plots for optimised network topologies, including electricity, gas and
 hydrogen networks, and regional generation, storage and conversion capacities
diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py
index 339a8457..cfa8e361 100644
--- a/scripts/plot_summary.py
+++ b/scripts/plot_summary.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Creates plots from summary CSV files.
 """
diff --git a/scripts/prepare_links_p_nom.py b/scripts/prepare_links_p_nom.py
index 7c63f3a4..7941b69a 100644
--- a/scripts/prepare_links_p_nom.py
+++ b/scripts/prepare_links_p_nom.py
@@ -4,7 +4,6 @@
 # SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Extracts capacities of HVDC links from `Wikipedia.
 
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 5937eb96..6129d3b5 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Adds all sector-coupling components to the network, including demand and supply
 technologies for the buildings, transport and industry sectors.
diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py
index b3a6cb4e..52884328 100644
--- a/scripts/retrieve_databundle.py
+++ b/scripts/retrieve_databundle.py
@@ -3,7 +3,6 @@
 # SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517935.svg
    :target: https://doi.org/10.5281/zenodo.3517935
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index 5b580bac..2b8cbb13 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Solves optimal operation and capacity for a network with the option to
 iteratively optimize while updating line reactances.
@@ -444,7 +443,7 @@ def add_operational_reserve_margin(n, sns, config):
 
     n.model.add_constraints(lhs >= rhs, name="reserve_margin")
 
-    # additional contraint that capacity is not exceeded
+    # additional constraint that capacity is not exceeded
     gen_i = n.generators.index
     ext_i = n.generators.query("p_nom_extendable").index
     fix_i = n.generators.query("not p_nom_extendable").index
diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py
index d16b60aa..25fe0753 100644
--- a/scripts/solve_operations_network.py
+++ b/scripts/solve_operations_network.py
@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
-
 """
 Solves linear optimal dispatch in hourly resolution using the capacities of
 previous capacity expansion in rule :mod:`solve_network`.

From 7b42a2251a10d5e03ab3bbe8d2ef76d44ea41f38 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Fri, 21 Apr 2023 10:28:49 +0200
Subject: [PATCH 025/176] renamem configs to config

---
 {configs => config}/config.default.yaml          | 0
 {configs => config}/test/config.electricity.yaml | 0
 {configs => config}/test/config.myopic.yaml      | 0
 {configs => config}/test/config.overnight.yaml   | 0
 4 files changed, 0 insertions(+), 0 deletions(-)
 rename {configs => config}/config.default.yaml (100%)
 rename {configs => config}/test/config.electricity.yaml (100%)
 rename {configs => config}/test/config.myopic.yaml (100%)
 rename {configs => config}/test/config.overnight.yaml (100%)

diff --git a/configs/config.default.yaml b/config/config.default.yaml
similarity index 100%
rename from configs/config.default.yaml
rename to config/config.default.yaml
diff --git a/configs/test/config.electricity.yaml b/config/test/config.electricity.yaml
similarity index 100%
rename from configs/test/config.electricity.yaml
rename to config/test/config.electricity.yaml
diff --git a/configs/test/config.myopic.yaml b/config/test/config.myopic.yaml
similarity index 100%
rename from configs/test/config.myopic.yaml
rename to config/test/config.myopic.yaml
diff --git a/configs/test/config.overnight.yaml b/config/test/config.overnight.yaml
similarity index 100%
rename from configs/test/config.overnight.yaml
rename to config/test/config.overnight.yaml

From 1f6f7a2039aa88e9bed9391d6dc841dee44d6df1 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Fri, 21 Apr 2023 10:41:44 +0200
Subject: [PATCH 026/176] replace config.yaml by config/config.yaml in doc and
 docstrings

---
 doc/configuration.rst               |  2 +-
 doc/costs.rst                       |  4 ++--
 doc/foresight.rst                   | 12 ++++++------
 doc/installation.rst                | 14 +++++++-------
 doc/introduction.rst                |  2 +-
 doc/release_notes.rst               |  2 +-
 doc/retrieve.rst                    |  6 +++---
 doc/spatial_resolution.rst          |  4 ++--
 doc/supply_demand.rst               |  2 +-
 doc/tutorial.rst                    |  6 +++---
 doc/tutorial_sector.rst             |  4 ++--
 scripts/_helpers.py                 |  2 +-
 scripts/add_electricity.py          |  2 +-
 scripts/add_extra_components.py     |  4 ++--
 scripts/base_network.py             |  2 +-
 scripts/build_bus_regions.py        |  2 +-
 scripts/build_cutout.py             |  2 +-
 scripts/build_electricity_demand.py |  2 +-
 scripts/build_hydro_profile.py      |  2 +-
 scripts/build_natura_raster.py      |  2 +-
 scripts/build_powerplants.py        |  2 +-
 scripts/build_renewable_profiles.py |  2 +-
 scripts/build_shapes.py             |  2 +-
 scripts/build_ship_raster.py        |  2 +-
 scripts/cluster_network.py          |  2 +-
 scripts/prepare_links_p_nom.py      |  2 +-
 scripts/prepare_network.py          |  2 +-
 scripts/retrieve_databundle.py      |  2 +-
 scripts/simplify_network.py         |  2 +-
 29 files changed, 48 insertions(+), 48 deletions(-)

diff --git a/doc/configuration.rst b/doc/configuration.rst
index ea8c81aa..17c8d9ac 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -9,7 +9,7 @@
 Configuration
 ##########################################
 
-PyPSA-Eur has several configuration options which are documented in this section and are collected in a ``config.yaml`` file located in the root directory. Users should copy the provided default configuration (``config.default.yaml``) and amend their own modifications and assumptions in the user-specific configuration file (``config.yaml``); confer installation instructions at :ref:`defaultconfig`.
+PyPSA-Eur has several configuration options which are documented in this section and are collected in a ``config/config.yaml`` file located in the root directory. Users should copy the provided default configuration (``config/config.default.yaml``) and amend their own modifications and assumptions in the user-specific configuration file (``config/config.yaml``); confer installation instructions at :ref:`defaultconfig`.
 
 .. _toplevel_cf:
 
diff --git a/doc/costs.rst b/doc/costs.rst
index 46b93482..21938ced 100644
--- a/doc/costs.rst
+++ b/doc/costs.rst
@@ -9,7 +9,7 @@ Techno-Economic Assumptions
 
 The database of cost assumptions is retrieved from the repository
 `PyPSA/technology-data <https://github.com/pypsa/technology-data>`_ and then
-saved to a file ``data/costs_{year}.csv``. The ``config.yaml`` provides options
+saved to a file ``data/costs_{year}.csv``. The ``config/config.yaml`` provides options
 to choose a reference year and use a specific version of the repository.
 
 .. literalinclude:: ../config.default.yaml
@@ -48,7 +48,7 @@ Modifying Assumptions
 =====================
 
 Some cost assumptions (e.g. marginal cost and capital cost) can be directly
-set in the ``config.yaml`` (cf. Section  :ref:`costs_cf`  in
+set in the ``config/config.yaml`` (cf. Section  :ref:`costs_cf`  in
 :ref:`config`). To change cost assumptions in more detail, make a copy of
 ``data/costs_{year}.csv`` and reference the new cost file in the ``Snakefile``:
 
diff --git a/doc/foresight.rst b/doc/foresight.rst
index f1ae2b38..9b821060 100644
--- a/doc/foresight.rst
+++ b/doc/foresight.rst
@@ -28,7 +28,7 @@ It does not affect the year for cost and technology assumptions, which is set se
   costs:
     year: 2030
 
-For running overnight scenarios, use in the ``config.yaml``:
+For running overnight scenarios, use in the ``config/config.yaml``:
 
 .. code:: yaml
 
@@ -44,7 +44,7 @@ Perfect foresight scenarios
   Perfect foresight is currently under development and not yet implemented.
 
 For running perfect foresight scenarios, in future versions you will be able to
-set in the ``config.yaml``:
+set in the ``config/config.yaml``:
 
 .. code:: yaml
 
@@ -94,13 +94,13 @@ evolve with the myopic approach:
 Configuration
 --------------
 
-For running myopic foresight transition scenarios, set in ``config.yaml``:
+For running myopic foresight transition scenarios, set in ``config/config.yaml``:
 
 .. code:: yaml
 
   foresight: myopic
 
-The following options included in the config.yaml file  are relevant for the
+The following options included in the ``config/config.yaml`` file  are relevant for the
 myopic code.
 
 The ``{planning_horizons}`` wildcard indicates the year in which the network is
@@ -163,7 +163,7 @@ Options
 The total carbon budget for the entire transition path can be indicated in the
 `sector_opts
 <https://github.com/PyPSA/pypsa-eur-sec/blob/f13902510010b734c510c38c4cae99356f683058/config.default.yaml#L25>`_
-in ``config.yaml``. The carbon budget can be split among the
+in ``config/config.yaml``. The carbon budget can be split among the
 ``planning_horizons`` following an exponential or beta decay. E.g. ``'cb40ex0'``
 splits a carbon budget equal to 40 Gt :math:`_{CO_2}` following an exponential
 decay whose initial linear growth rate r is zero. They can also follow some
@@ -218,7 +218,7 @@ add_brownfield for the remaining planning_horizons.
    ``results/run_name/networks/prenetworks-brownfield``.
 
 Steps 2 and 3 are solved recursively for all the planning_horizons included in
-``config.yaml``.
+``config/config.yaml``.
 
 Rule overview
 --------------
diff --git a/doc/installation.rst b/doc/installation.rst
index ed67f27d..b6fd574e 100644
--- a/doc/installation.rst
+++ b/doc/installation.rst
@@ -119,19 +119,19 @@ Handling Configuration Files
 ============================
 
 PyPSA-Eur has several configuration options that must be specified in a
-``config.yaml`` file located in the root directory. An example configuration
-``config.default.yaml`` is maintained in the repository, which will be used to
-automatically create your customisable ``config.yaml`` on first use. More
+``config/config.yaml`` file located in the root directory. An example configuration
+``config/config.default.yaml`` is maintained in the repository, which will be used to
+automatically create your customisable ``config/config.yaml`` on first use. More
 details on the configuration options are in :ref:`config`.
 
 You can also use ``snakemake`` to specify another file, e.g.
-``config.mymodifications.yaml``, to update the settings of the ``config.yaml``.
+``config/config.mymodifications.yaml``, to update the settings of the ``config/config.yaml``.
 
 .. code:: bash
 
-    .../pypsa-eur % snakemake -call --configfile config.mymodifications.yaml
+    .../pypsa-eur % snakemake -call --configfile config/config.mymodifications.yaml
 
 .. warning::
-    Users are advised to regularly check their own ``config.yaml`` against changes
-    in the ``config.default.yaml`` when pulling a new version from the remote
+    Users are advised to regularly check their own ``config/config.yaml`` against changes
+    in the ``config/config.default.yaml`` when pulling a new version from the remote
     repository.
diff --git a/doc/introduction.rst b/doc/introduction.rst
index 0ae038aa..df060723 100644
--- a/doc/introduction.rst
+++ b/doc/introduction.rst
@@ -74,7 +74,7 @@ what data to retrieve and what files to produce. Details are explained in
 :ref:`wildcards` and :ref:`scenario`.
 
 The model also has several further configuration options collected in the
-``config.yaml`` file located in the root directory, which that are not part of
+``config/config.yaml`` file located in the root directory, which that are not part of
 the scenarios. Options are explained in :ref:`config`.
 
 Folder Structure
diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index 3d05f27c..d9fc5378 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -339,7 +339,7 @@ PyPSA-Eur 0.5.0 (27th July 2022)
 * Network building is made deterministic by supplying a fixed random state to
   network clustering routines.
 
-* Clustering strategies for generator and bus attributes can now be specified directly in the ``config.yaml``.
+* Clustering strategies for generator and bus attributes can now be specified directly in the ``config/config.yaml``.
 
 * Iterative solving with impedance updates is skipped if there are no expandable
   lines.
diff --git a/doc/retrieve.rst b/doc/retrieve.rst
index c06b330f..e4f47dec 100644
--- a/doc/retrieve.rst
+++ b/doc/retrieve.rst
@@ -42,7 +42,7 @@ The :ref:`tutorial` uses a smaller cutout than required for the full model (30 M
         build_cutout:
 
 .. seealso::
-    Documentation of the configuration file ``config.yaml`` at
+    Documentation of the configuration file ``config/config.yaml`` at
     :ref:`toplevel_cf`
 
 **Outputs**
@@ -69,7 +69,7 @@ This rule, as a substitute for :mod:`build_natura_raster`, downloads an already
         build_natura_raster:
 
 .. seealso::
-    Documentation of the configuration file ``config.yaml`` at
+    Documentation of the configuration file ``config/config.yaml`` at
     :ref:`toplevel_cf`
 
 **Outputs**
@@ -111,7 +111,7 @@ This rule downloads techno-economic assumptions from the `technology-data reposi
         version:
 
 .. seealso::
-    Documentation of the configuration file ``config.yaml`` at
+    Documentation of the configuration file ``config/config.yaml`` at
     :ref:`costs_cf`
 
 **Outputs**
diff --git a/doc/spatial_resolution.rst b/doc/spatial_resolution.rst
index bbe5da05..0293a5ce 100644
--- a/doc/spatial_resolution.rst
+++ b/doc/spatial_resolution.rst
@@ -11,7 +11,7 @@ Spatial resolution
 
 The default nodal resolution of the model follows the electricity generation and transmission model `PyPSA-Eur <https://github.com/PyPSA/pypsa-eur>`_, which clusters down the electricity transmission substations in each European country based on the k-means algorithm (See `cluster_network <https://pypsa-eur.readthedocs.io/en/latest/simplification/cluster_network.html#rule-cluster-network>`_ for a complete explanation). This gives nodes which correspond to major load and generation centres (typically cities).
 
-The total number of nodes for Europe is set in the ``config.yaml`` file under ``clusters``. The number of nodes can vary between 37, the number of independent countries / synchronous areas, and several hundred. With 200-300 nodes the model needs 100-150 GB RAM to solve with a commercial solver like Gurobi.
+The total number of nodes for Europe is set in the ``config/config.yaml`` file under ``clusters``. The number of nodes can vary between 37, the number of independent countries / synchronous areas, and several hundred. With 200-300 nodes the model needs 100-150 GB RAM to solve with a commercial solver like Gurobi.
 
 Exemplary unsolved network clustered to 512 nodes:
 
@@ -21,7 +21,7 @@ Exemplary unsolved network clustered to 37 nodes:
 
 .. image:: ../graphics/elec_s_37.png
 
-The total number of nodes for Europe is set in the config.yaml file under `clusters <https://github.com/PyPSA/pypsa-eur-sec/blob/3daff49c9999ba7ca7534df4e587e1d516044fc3/config.default.yaml#L20>`_.  The number of nodes can vary between 37, the number of independent countries/synchronous areas, and several hundred. With 200-300 nodes, the model needs 100-150 GB RAM to solve with a commercial solver like Gurobi.
+The total number of nodes for Europe is set in the ``config/config.yaml`` file under `clusters <https://github.com/PyPSA/pypsa-eur-sec/blob/3daff49c9999ba7ca7534df4e587e1d516044fc3/config.default.yaml#L20>`_.  The number of nodes can vary between 37, the number of independent countries/synchronous areas, and several hundred. With 200-300 nodes, the model needs 100-150 GB RAM to solve with a commercial solver like Gurobi.
 Not all of the sectors are at the full nodal resolution, and some demand for some sectors is distributed to nodes using heuristics that need to be corrected. Some networks are copper-plated to reduce computational times.
 
 Here are some examples of how spatial resolution is set for different sectors in PyPSA-Eur-Sec:
diff --git a/doc/supply_demand.rst b/doc/supply_demand.rst
index ed35bc38..16242405 100644
--- a/doc/supply_demand.rst
+++ b/doc/supply_demand.rst
@@ -189,7 +189,7 @@ higher costs and higher efficiency gains. They are added by step-wise
 linearisation in form of two additional generations in
 the  `prepare_sector_network.py <https://github.com/PyPSA/pypsa-eur-sec/blob/3daff49c9999ba7ca7534df4e587e1d516044fc3/scripts/prepare_sector_network.py#L1600>`_  script.
 
-Settings in the config.yaml concerning the endogenously optimisation of building
+Settings in the ``config/config.yaml`` concerning the endogenously optimisation of building
 renovation include `cost factor <https://github.com/PyPSA/pypsa-eur-sec/blob/3daff49c9999ba7ca7534df4e587e1d516044fc3/config.default.yaml#L223>`_, `interest rate <https://github.com/PyPSA/pypsa-eur-sec/blob/3daff49c9999ba7ca7534df4e587e1d516044fc3/config.default.yaml#L224>`_, `annualised cost <https://github.com/PyPSA/pypsa-eur-sec/blob/3daff49c9999ba7ca7534df4e587e1d516044fc3/config.default.yaml#L225>`_, `tax weighting <https://github.com/PyPSA/pypsa-eur-sec/blob/3daff49c9999ba7ca7534df4e587e1d516044fc3/config.default.yaml#L226>`_, and `construction index <https://github.com/PyPSA/pypsa-eur-sec/blob/3daff49c9999ba7ca7534df4e587e1d516044fc3/config.default.yaml#L227>`_.
 
 Further information are given in the study by Zeyen et al. : `Mitigating heat demand peaks in buildings in a highly renewable European energy system, (2021) <https://arxiv.org/abs/2012.01831>`_.
diff --git a/doc/tutorial.rst b/doc/tutorial.rst
index 82ccc55a..f396b82c 100644
--- a/doc/tutorial.rst
+++ b/doc/tutorial.rst
@@ -26,7 +26,7 @@ local machine. The tutorial will cover examples on how to configure and
 customise the PyPSA-Eur model and run the ``snakemake`` workflow step by step
 from network creation to the solved network. The configuration for the tutorial
 is located at ``test/config.electricity.yaml``. It includes parts deviating from
-the default config file ``config.default.yaml``. To run the tutorial with this
+the default config file ``config/config.default.yaml``. To run the tutorial with this
 configuration, execute
 
 .. code:: bash
@@ -282,7 +282,7 @@ For example, you can explore the evolution of the PyPSA networks by running
 #. ``snakemake resources/networks/elec_s_6.nc -call --configfile test/config.electricity.yaml``
 #. ``snakemake resources/networks/elec_s_6_ec_lcopt_Co2L-24H.nc -call --configfile test/config.electricity.yaml``
 
-To run all combinations of wildcard values provided in the ``config.yaml`` under ``scenario:``,
+To run all combinations of wildcard values provided in the ``config/config.yaml`` under ``scenario:``,
 you can use the collection rule ``solve_elec_networks``.
 
 .. code:: bash
@@ -290,7 +290,7 @@ you can use the collection rule ``solve_elec_networks``.
     snakemake -call solve_elec_networks --configfile test/config.electricity.yaml
 
 If you now feel confident and want to tackle runs with larger temporal and
-spatial scope, clean-up the repository and after modifying the ``config.yaml`` file
+spatial scope, clean-up the repository and after modifying the ``config/config.yaml`` file
 target the collection rule ``solve_elec_networks`` again without providing the test
 configuration file.
 
diff --git a/doc/tutorial_sector.rst b/doc/tutorial_sector.rst
index 1cbd6779..039271db 100644
--- a/doc/tutorial_sector.rst
+++ b/doc/tutorial_sector.rst
@@ -29,7 +29,7 @@ Overnight Scenarios
 Configuration
 -------------
 
-The default configuration file (``config.default.yaml``) is set up for running
+The default configuration file (``config/config.default.yaml``) is set up for running
 overnight scenarios. Running a sector-coupled model unlocks many further
 configuration options. In the example below, we say that the gas network should
 be added and spatially resolved. We also say that the existing gas network may
@@ -513,7 +513,7 @@ Scaling-Up
 ==========
 
 If you now feel confident and want to tackle runs with larger temporal, technological and
-spatial scope, clean-up the repository and after modifying the ``config.yaml`` file
+spatial scope, clean-up the repository and after modifying the ``config/config.yaml`` file
 target the collection rule ``all`` again without providing the test
 configuration file.
 
diff --git a/scripts/_helpers.py b/scripts/_helpers.py
index dc10eac1..a67fb105 100644
--- a/scripts/_helpers.py
+++ b/scripts/_helpers.py
@@ -83,7 +83,7 @@ def load_network(import_name=None, custom_components=None):
     custom_components : dict
         Dictionary listing custom components.
         For using ``snakemake.config['override_components']``
-        in ``config.yaml`` define:
+        in ``config/config.yaml`` define:
 
         .. code:: yaml
 
diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index ef6ff347..1d32bce1 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -41,7 +41,7 @@ Relevant Settings
         length_factor:
 
 .. seealso::
-    Documentation of the configuration file ``config.yaml`` at :ref:`costs_cf`,
+    Documentation of the configuration file ``config/config.yaml`` at :ref:`costs_cf`,
     :ref:`electricity_cf`, :ref:`load_cf`, :ref:`renewable_cf`, :ref:`lines_cf`
 
 Inputs
diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py
index b507148d..020370e5 100644
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@@ -27,7 +27,7 @@ Relevant Settings
             Store:
 
 .. seealso::
-    Documentation of the configuration file ``config.yaml`` at :ref:`costs_cf`,
+    Documentation of the configuration file ``config/config.yaml`` at :ref:`costs_cf`,
     :ref:`electricity_cf`
 
 Inputs
@@ -44,7 +44,7 @@ Outputs
 Description
 -----------
 
-The rule :mod:`add_extra_components` attaches additional extendable components to the clustered and simplified network. These can be configured in the ``config.yaml`` at ``electricity: extendable_carriers:``. It processes ``networks/elec_s{simpl}_{clusters}.nc`` to build ``networks/elec_s{simpl}_{clusters}_ec.nc``, which in contrast to the former (depending on the configuration) contain with **zero** initial capacity
+The rule :mod:`add_extra_components` attaches additional extendable components to the clustered and simplified network. These can be configured in the ``config/config.yaml`` at ``electricity: extendable_carriers:``. It processes ``networks/elec_s{simpl}_{clusters}.nc`` to build ``networks/elec_s{simpl}_{clusters}_ec.nc``, which in contrast to the former (depending on the configuration) contain with **zero** initial capacity
 
 - ``StorageUnits`` of carrier 'H2' and/or 'battery'. If this option is chosen, every bus is given an extendable ``StorageUnit`` of the corresponding carrier. The energy and power capacities are linked through a parameter that specifies the energy capacity as maximum hours at full dispatch power and is configured in ``electricity: max_hours:``. This linkage leads to one investment variable per storage unit. The default ``max_hours`` lead to long-term hydrogen and short-term battery storage units.
 
diff --git a/scripts/base_network.py b/scripts/base_network.py
index 24097b1d..53b4dca3 100644
--- a/scripts/base_network.py
+++ b/scripts/base_network.py
@@ -38,7 +38,7 @@ Relevant Settings
         type:
 
 .. seealso::
-    Documentation of the configuration file ``config.yaml`` at
+    Documentation of the configuration file ``config/config.yaml`` at
     :ref:`snapshots_cf`, :ref:`toplevel_cf`, :ref:`electricity_cf`, :ref:`load_cf`,
     :ref:`lines_cf`, :ref:`links_cf`, :ref:`transformers_cf`
 
diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py
index cfbf90f7..ee90eef6 100644
--- a/scripts/build_bus_regions.py
+++ b/scripts/build_bus_regions.py
@@ -14,7 +14,7 @@ Relevant Settings
     countries:
 
 .. seealso::
-    Documentation of the configuration file ``config.yaml`` at
+    Documentation of the configuration file ``config/config.yaml`` at
     :ref:`toplevel_cf`
 
 Inputs
diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py
index eefa881b..365797d2 100644
--- a/scripts/build_cutout.py
+++ b/scripts/build_cutout.py
@@ -25,7 +25,7 @@ Relevant Settings
             {cutout}:
 
 .. seealso::
-    Documentation of the configuration file ``config.yaml`` at
+    Documentation of the configuration file ``config/config.yaml`` at
     :ref:`atlite_cf`
 
 Inputs
diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py
index 30f24f5c..b86b4a5f 100755
--- a/scripts/build_electricity_demand.py
+++ b/scripts/build_electricity_demand.py
@@ -25,7 +25,7 @@ Relevant Settings
 
 
 .. seealso::
-    Documentation of the configuration file ``config.yaml`` at
+    Documentation of the configuration file ``config/config.yaml`` at
     :ref:`load_cf`
 
 Inputs
diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py
index 9adf7112..0e8cfa27 100644
--- a/scripts/build_hydro_profile.py
+++ b/scripts/build_hydro_profile.py
@@ -20,7 +20,7 @@ Relevant Settings
             clip_min_inflow:
 
 .. seealso::
-    Documentation of the configuration file ``config.yaml`` at
+    Documentation of the configuration file ``config/config.yaml`` at
     :ref:`toplevel_cf`, :ref:`renewable_cf`
 
 Inputs
diff --git a/scripts/build_natura_raster.py b/scripts/build_natura_raster.py
index 12011cd0..3cd62fd9 100644
--- a/scripts/build_natura_raster.py
+++ b/scripts/build_natura_raster.py
@@ -18,7 +18,7 @@ Relevant Settings
             cutout:
 
 .. seealso::
-    Documentation of the configuration file ``config.yaml`` at
+    Documentation of the configuration file ``config/config.yaml`` at
     :ref:`renewable_cf`
 
 Inputs
diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py
index a8a65249..9ca67a53 100755
--- a/scripts/build_powerplants.py
+++ b/scripts/build_powerplants.py
@@ -21,7 +21,7 @@ Relevant Settings
       custom_powerplants:
 
 .. seealso::
-    Documentation of the configuration file ``config.yaml`` at
+    Documentation of the configuration file ``config/config.yaml`` at
     :ref:`electricity`
 
 Inputs
diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py
index 6f03d746..c0288aee 100644
--- a/scripts/build_renewable_profiles.py
+++ b/scripts/build_renewable_profiles.py
@@ -42,7 +42,7 @@ Relevant settings
             resource:
 
 .. seealso::
-    Documentation of the configuration file ``config.yaml`` at
+    Documentation of the configuration file ``config/config.yaml`` at
     :ref:`snapshots_cf`, :ref:`atlite_cf`, :ref:`renewable_cf`
 
 Inputs
diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py
index fda7613f..50d21e12 100644
--- a/scripts/build_shapes.py
+++ b/scripts/build_shapes.py
@@ -15,7 +15,7 @@ Relevant Settings
     countries:
 
 .. seealso::
-    Documentation of the configuration file ``config.yaml`` at
+    Documentation of the configuration file ``config/config.yaml`` at
     :ref:`toplevel_cf`
 
 Inputs
diff --git a/scripts/build_ship_raster.py b/scripts/build_ship_raster.py
index abfcdfa1..90e006b0 100644
--- a/scripts/build_ship_raster.py
+++ b/scripts/build_ship_raster.py
@@ -19,7 +19,7 @@ Relevant Settings
             cutout:
 
 .. seealso::
-    Documentation of the configuration file ``config.yaml`` at
+    Documentation of the configuration file ``config/config.yaml`` at
     :ref:`renewable_cf`
 
 Inputs
diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py
index e2cba28f..7572d3b3 100644
--- a/scripts/cluster_network.py
+++ b/scripts/cluster_network.py
@@ -27,7 +27,7 @@ Relevant Settings
         length_factor:
 
 .. seealso::
-    Documentation of the configuration file ``config.yaml`` at
+    Documentation of the configuration file ``config/config.yaml`` at
     :ref:`toplevel_cf`, :ref:`renewable_cf`, :ref:`solving_cf`, :ref:`lines_cf`
 
 Inputs
diff --git a/scripts/prepare_links_p_nom.py b/scripts/prepare_links_p_nom.py
index 7941b69a..4b915d22 100644
--- a/scripts/prepare_links_p_nom.py
+++ b/scripts/prepare_links_p_nom.py
@@ -18,7 +18,7 @@ Relevant Settings
         prepare_links_p_nom:
 
 .. seealso::
-    Documentation of the configuration file ``config.yaml`` at
+    Documentation of the configuration file ``config/config.yaml`` at
     :ref:`toplevel_cf`
 
 Inputs
diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index 79b5c9d4..14a003ae 100755
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -34,7 +34,7 @@ Relevant Settings
         max_hours:
 
 .. seealso::
-    Documentation of the configuration file ``config.yaml`` at
+    Documentation of the configuration file ``config/config.yaml`` at
     :ref:`costs_cf`, :ref:`electricity_cf`
 
 Inputs
diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py
index 52884328..0c6a7feb 100644
--- a/scripts/retrieve_databundle.py
+++ b/scripts/retrieve_databundle.py
@@ -23,7 +23,7 @@ The :ref:`tutorial` uses a smaller `data bundle <https://zenodo.org/record/35179
     tutorial:
 
 .. seealso::
-    Documentation of the configuration file ``config.yaml`` at
+    Documentation of the configuration file ``config/config.yaml`` at
     :ref:`toplevel_cf`
 
 **Outputs**
diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 58c4ad17..2be8c36a 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -40,7 +40,7 @@ Relevant Settings
             name:
 
 .. seealso::
-    Documentation of the configuration file ``config.yaml`` at
+    Documentation of the configuration file ``config/config.yaml`` at
     :ref:`costs_cf`, :ref:`electricity_cf`, :ref:`renewable_cf`,
     :ref:`lines_cf`, :ref:`links_cf`, :ref:`solving_cf`
 

From 97bdbe636cddc3cd1ffc51d01bc09014291abdda Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Fri, 21 Apr 2023 11:05:18 +0200
Subject: [PATCH 027/176] follow-up: rename configs to config

---
 .github/workflows/ci.yaml | 6 +++---
 Snakefile                 | 6 +++---
 rules/postprocess.smk     | 4 ++--
 rules/solve_myopic.smk    | 2 +-
 rules/solve_overnight.smk | 4 ++--
 scripts/copy_config.py    | 4 ++--
 6 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 2e7de2d3..f0cdfca1 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -103,9 +103,9 @@ jobs:
       run: |
         conda activate pypsa-eur
         conda list
-        snakemake -call solve_elec_networks --configfile configs/test/config.electricity.yaml --rerun-triggers=mtime
-        snakemake -call all --configfile configs/test/config.overnight.yaml --rerun-triggers=mtime
-        snakemake -call all --configfile configs/test/config.myopic.yaml --rerun-triggers=mtime
+        snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime
+        snakemake -call all --configfile config/test/config.overnight.yaml --rerun-triggers=mtime
+        snakemake -call all --configfile config/test/config.myopic.yaml --rerun-triggers=mtime
 
     - name: Upload artifacts
       uses: actions/upload-artifact@v3
diff --git a/Snakefile b/Snakefile
index 48529908..27ed4dfb 100644
--- a/Snakefile
+++ b/Snakefile
@@ -14,11 +14,11 @@ from snakemake.utils import min_version
 min_version("7.7")
 
 
-if not exists("configs/config.yaml"):
-    copyfile("configs/config.default.yaml", "configs/config.yaml")
+if not exists("config/config.yaml"):
+    copyfile("config/config.default.yaml", "config/config.yaml")
 
 
-configfile: "configs/config.yaml"
+configfile: "config/config.yaml"
 
 
 COSTS = f"data/costs_{config['costs']['year']}.csv"
diff --git a/rules/postprocess.smk b/rules/postprocess.smk
index d5095358..fae0f856 100644
--- a/rules/postprocess.smk
+++ b/rules/postprocess.smk
@@ -37,7 +37,7 @@ rule copy_config:
     params:
         RDIR=RDIR,
     output:
-        RESULTS + "configs/config.yaml",
+        RESULTS + "config/config.yaml",
     threads: 1
     resources:
         mem_mb=1000,
@@ -51,7 +51,7 @@ rule copy_config:
 
 rule copy_conda_env:
     output:
-        RESULTS + "configs/environment.yaml",
+        RESULTS + "config/environment.yaml",
     threads: 1
     resources:
         mem_mb=500,
diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk
index 041bee84..f10d8157 100644
--- a/rules/solve_myopic.smk
+++ b/rules/solve_myopic.smk
@@ -79,7 +79,7 @@ rule solve_sector_network_myopic:
         network=RESULTS
         + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
         costs="data/costs_{planning_horizons}.csv",
-        config=RESULTS + "configs/config.yaml",
+        config=RESULTS + "config/config.yaml",
     output:
         RESULTS
         + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk
index c39662ec..c2e103e5 100644
--- a/rules/solve_overnight.smk
+++ b/rules/solve_overnight.smk
@@ -9,8 +9,8 @@ rule solve_sector_network:
         network=RESULTS
         + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
         costs="data/costs_{}.csv".format(config["costs"]["year"]),
-        config=RESULTS + "configs/config.yaml",
-        #env=RDIR + 'configs/environment.yaml',
+        config=RESULTS + "config/config.yaml",
+        #env=RDIR + 'config/environment.yaml',
     output:
         RESULTS
         + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
diff --git a/scripts/copy_config.py b/scripts/copy_config.py
index 084601ed..79d2e32b 100644
--- a/scripts/copy_config.py
+++ b/scripts/copy_config.py
@@ -12,7 +12,7 @@ from shutil import copy
 import yaml
 
 files = {
-    "configs/config.yaml": "config.yaml",
+    "config/config.yaml": "config.yaml",
     "Snakefile": "Snakefile",
     "scripts/solve_network.py": "solve_network.py",
     "scripts/prepare_sector_network.py": "prepare_sector_network.py",
@@ -24,7 +24,7 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("copy_config")
 
-    basepath = Path(f"results/{snakemake.params.RDIR}configs/")
+    basepath = Path(f"results/{snakemake.params.RDIR}config/")
 
     for f, name in files.items():
         copy(f, basepath / name)

From 8c19a74fa5ba9056ffb559f7b259d756175998f5 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Fri, 21 Apr 2023 11:33:06 +0200
Subject: [PATCH 028/176] add release notes fix config references in
 configuration.rst

---
 doc/configuration.rst | 54 +++++++++++++++++++++----------------------
 doc/release_notes.rst |  3 +++
 2 files changed, 30 insertions(+), 27 deletions(-)

diff --git a/doc/configuration.rst b/doc/configuration.rst
index 17c8d9ac..ee61c018 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -16,7 +16,7 @@ PyPSA-Eur has several configuration options which are documented in this section
 Top-level configuration
 =======================
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :lines: 5-11,18-19,62,80-90
 
@@ -37,7 +37,7 @@ investment changes as more ambitious greenhouse-gas emission reduction targets a
 
 The ``run`` section is used for running and storing scenarios with different configurations which are not covered by :ref:`wildcards`. It determines the path at which resources, networks and results are stored. Therefore the user can run different configurations within the same directory. If a run with a non-empty name should use cutouts shared across runs, set ``shared_cutouts`` to `true`.
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: run:
    :end-before: foresight:
@@ -76,7 +76,7 @@ An exemplary dependency graph (starting from the simplification rules) then look
 
 .. image:: img/scenarios.png
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: scenario:
    :end-before: countries:
@@ -93,7 +93,7 @@ An exemplary dependency graph (starting from the simplification rules) then look
 
 Specifies the temporal range to build an energy system model for as arguments to `pandas.date_range <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.date_range.html>`_
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: snapshots:
    :end-before: enable:
@@ -110,7 +110,7 @@ Specifies the temporal range to build an energy system model for as arguments to
 
 Switches for some rules and optional features.
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: enable:
    :end-before: co2_budget:
@@ -125,7 +125,7 @@ Switches for some rules and optional features.
 ``electricity``
 ===============
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: electricity:
    :end-before: atlite:
@@ -142,7 +142,7 @@ Switches for some rules and optional features.
 
 Define and specify the ``atlite.Cutout`` used for calculating renewable potentials and time-series. All options except for ``features`` are directly used as `cutout parameters <https://atlite.readthedocs.io/en/latest/ref_api.html#cutout>`_.
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: atlite:
    :end-before: renewable:
@@ -160,7 +160,7 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia
 ``onwind``
 ----------
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: renewable:
    :end-before:   offwind-ac:
@@ -173,7 +173,7 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia
 ``offwind-ac``
 --------------
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at:   offwind-ac:
    :end-before:   offwind-dc:
@@ -186,7 +186,7 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia
 ``offwind-dc``
 ---------------
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at:   offwind-dc:
    :end-before:   solar:
@@ -199,7 +199,7 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia
 ``solar``
 ---------------
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at:   solar:
    :end-before:   hydro:
@@ -212,7 +212,7 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia
 ``hydro``
 ---------------
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at:   hydro:
    :end-before: conventional:
@@ -234,7 +234,7 @@ with country specific values. Then, the values are read in and applied to all
 generators of the given carrier in the given country. Note that the value(s)
 overwrite the existing values.
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at:   conventional:
    :end-before: lines:
@@ -247,7 +247,7 @@ overwrite the existing values.
 ``lines``
 =============
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: lines:
    :end-before: links:
@@ -262,7 +262,7 @@ overwrite the existing values.
 ``links``
 =============
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: links:
    :end-before: transformers:
@@ -277,7 +277,7 @@ overwrite the existing values.
 ``transformers``
 ================
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: transformers:
    :end-before: load:
@@ -292,7 +292,7 @@ overwrite the existing values.
 ``load``
 =============
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-after:   type:
    :end-at:   scaling_factor:
@@ -307,7 +307,7 @@ overwrite the existing values.
 ``costs``
 =============
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: costs:
    :end-before: clustering:
@@ -323,7 +323,7 @@ overwrite the existing values.
 ``clustering``
 ==============
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: clustering:
    :end-before: solving:
@@ -345,7 +345,7 @@ overwrite the existing values.
 .. warning::
    More comprehensive documentation for this segment will be released soon.
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: energy:
    :end-before: biomass:
@@ -362,7 +362,7 @@ overwrite the existing values.
 .. warning::
    More comprehensive documentation for this segment will be released soon.
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: biomass:
    :end-before: solar_thermal:
@@ -378,7 +378,7 @@ overwrite the existing values.
 .. warning::
    More comprehensive documentation for this segment will be released soon.
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: solar_thermal:
    :end-before: existing_capacities:
@@ -394,7 +394,7 @@ overwrite the existing values.
 .. warning::
    More comprehensive documentation for this segment will be released soon.
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: existing_capacities:
    :end-before: sector:
@@ -410,7 +410,7 @@ overwrite the existing values.
 .. warning::
    More comprehensive documentation for this segment will be released soon.
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: sector:
    :end-before: industry:
@@ -426,7 +426,7 @@ overwrite the existing values.
 .. warning::
    More comprehensive documentation for this segment will be released soon.
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: industry:
    :end-before: costs:
@@ -436,7 +436,7 @@ overwrite the existing values.
 ``solving``
 =============
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: solving:
    :end-before: plotting:
@@ -459,7 +459,7 @@ overwrite the existing values.
 .. warning::
    More comprehensive documentation for this segment will be released soon.
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: plotting:
 
diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index d9fc5378..f859646b 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -10,6 +10,9 @@ Release Notes
 Upcoming Release
 ================
 
+
+* **Important:** The configuration files are now located in the ``config`` directory. This counts for ``config.default.yaml``, ``config.yaml`` as well as the test configuration files which are now located in ``config/test``. Config files that are still in the root directory will be ignored.
+
 * Bugfix: Correct typo in the CPLEX solver configuration in ``config.default.yaml``.
 
 * Renamed script file from PyPSA-EUR ``build_load_data`` to ``build_electricity_demand``.

From f2e53b59bf92fc477233c7919dd61a2f94a95ae0 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sat, 22 Apr 2023 09:44:13 +0200
Subject: [PATCH 029/176] add plain hydrogen turbine option for
 re-electrification

---
 config/config.default.yaml        |  3 +++
 doc/release_notes.rst             |  4 +++
 scripts/prepare_sector_network.py | 45 ++++++++++++++++++++++---------
 3 files changed, 40 insertions(+), 12 deletions(-)

diff --git a/config/config.default.yaml b/config/config.default.yaml
index 1305456a..10e6a6ed 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -469,6 +469,8 @@ sector:
   dac: true
   co2_vent: false
   allam_cycle: false
+  hydrogen_fuel_cell: true
+  hydrogen_turbine: false
   SMR: true
   regional_co2_sequestration_potential:
     enable: false  # enable regionally resolved geological co2 storage potential
@@ -902,6 +904,7 @@ plotting:
     H2 pipeline: '#f081dc'
     H2 pipeline retrofitted: '#ba99b5'
     H2 Fuel Cell: '#c251ae'
+    H2 turbine: '#991f83'
     H2 Electrolysis: '#ff29d9'
     # ammonia
     NH3: '#46caf0'
diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index f859646b..d81bee07 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -17,6 +17,10 @@ Upcoming Release
 
 * Renamed script file from PyPSA-EUR ``build_load_data`` to ``build_electricity_demand``.
 
+* Add plain hydrogen turbine as additional re-electrification option besides
+  hydrogen fuel cell. Add switches for both re-electrification options under
+  ``sector: hydrogen_turbine:`` and ``sector: hydrogen_fuel_cell:``.
+
 PyPSA-Eur 0.8.0 (18th March 2023)
 =================================
 
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 6129d3b5..e737867e 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -1067,18 +1067,39 @@ def add_storage_and_grids(n, costs):
         lifetime=costs.at["electrolysis", "lifetime"],
     )
 
-    n.madd(
-        "Link",
-        nodes + " H2 Fuel Cell",
-        bus0=nodes + " H2",
-        bus1=nodes,
-        p_nom_extendable=True,
-        carrier="H2 Fuel Cell",
-        efficiency=costs.at["fuel cell", "efficiency"],
-        capital_cost=costs.at["fuel cell", "fixed"]
-        * costs.at["fuel cell", "efficiency"],  # NB: fixed cost is per MWel
-        lifetime=costs.at["fuel cell", "lifetime"],
-    )
+    if options["hydrogen_fuel_cell"]:
+
+        logger.info("Adding hydrogen fuel cell for re-electrification.")
+
+        n.madd(
+            "Link",
+            nodes + " H2 Fuel Cell",
+            bus0=nodes + " H2",
+            bus1=nodes,
+            p_nom_extendable=True,
+            carrier="H2 Fuel Cell",
+            efficiency=costs.at["fuel cell", "efficiency"],
+            capital_cost=costs.at["fuel cell", "fixed"]
+            * costs.at["fuel cell", "efficiency"],  # NB: fixed cost is per MWel
+            lifetime=costs.at["fuel cell", "lifetime"],
+        )
+
+    if options["hydrogen_turbine"]:
+
+        logger.info("Adding hydrogen turbine for re-electrification.")
+
+        n.madd(
+            "Link",
+            nodes + " H2 turbine",
+            bus0=nodes + " H2",
+            bus1=nodes,
+            p_nom_extendable=True,
+            carrier="H2 turbine",
+            efficiency=costs.at["OCGT", "efficiency"],
+            capital_cost=costs.at["OCGT", "fixed"]
+            * costs.at["OCGT", "efficiency"],  # NB: fixed cost is per MWel
+            lifetime=costs.at["OCGT", "lifetime"],
+        )
 
     cavern_types = snakemake.config["sector"]["hydrogen_underground_storage_locations"]
     h2_caverns = pd.read_csv(snakemake.input.h2_cavern, index_col=0)

From 0a3c177f4b6083a00f75810340cf7570f373bb15 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sat, 22 Apr 2023 07:44:57 +0000
Subject: [PATCH 030/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/prepare_sector_network.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index e737867e..90891ad5 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -1068,7 +1068,6 @@ def add_storage_and_grids(n, costs):
     )
 
     if options["hydrogen_fuel_cell"]:
-
         logger.info("Adding hydrogen fuel cell for re-electrification.")
 
         n.madd(
@@ -1085,7 +1084,6 @@ def add_storage_and_grids(n, costs):
         )
 
     if options["hydrogen_turbine"]:
-
         logger.info("Adding hydrogen turbine for re-electrification.")
 
         n.madd(

From ad9a67509f6a2d3f62ae4dd2f5685dcac149af50 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Mon, 24 Apr 2023 05:05:43 +0200
Subject: [PATCH 031/176] Add carrier "AC" and "DC" in base_network.py

---
 scripts/base_network.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/base_network.py b/scripts/base_network.py
index 53b4dca3..ecf21e1f 100644
--- a/scripts/base_network.py
+++ b/scripts/base_network.py
@@ -714,6 +714,7 @@ def base_network(
     n.name = "PyPSA-Eur"
 
     n.set_snapshots(pd.date_range(freq="h", **config["snapshots"]))
+    n.madd("Carrier",["AC","DC"])
 
     n.import_components_from_dataframe(buses, "Bus")
     n.import_components_from_dataframe(lines, "Line")

From e20e7bf844fa9300e242c43cd3c4879d9d8b5fdc Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Mon, 24 Apr 2023 12:33:55 +0200
Subject: [PATCH 032/176] fix addition of load shedding

---
 doc/configtables/solving.csv | 2 +-
 scripts/solve_network.py     | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/doc/configtables/solving.csv b/doc/configtables/solving.csv
index d30029f4..cba28cbe 100644
--- a/doc/configtables/solving.csv
+++ b/doc/configtables/solving.csv
@@ -1,7 +1,7 @@
 ,Unit,Values,Description
 options,,,
 -- formulation,--,"Any of {'angles', 'kirchhoff', 'cycles', 'ptdf'}","Specifies which variant of linearized power flow formulations to use in the optimisation problem. Recommended is 'kirchhoff'. Explained in `this article <https://arxiv.org/abs/1704.01881>`_."
--- load_shedding,bool,"{'true','false'}","Add generators with a prohibitively high marginal cost to simulate load shedding and avoid problem infeasibilities."
+-- load_shedding,bool/float,"{'true','false', float}","Add generators with very high marginal cost to simulate load shedding and avoid problem infeasibilities. If load shedding is a float, it denotes the marginal cost in EUR/kWh."
 -- noisy_costs,bool,"{'true','false'}","Add random noise to marginal cost of generators by :math:`\mathcal{U}(0.009,0,011)` and capital cost of lines and links by :math:`\mathcal{U}(0.09,0,11)`."
 -- min_iterations,--,int,"Minimum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run."
 -- max_iterations,--,int,"Maximum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run."
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index 2b8cbb13..fcdc4ed9 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -151,7 +151,8 @@ def prepare_network(n, solve_opts=None, config=None):
         ):
             df.where(df > solve_opts["clip_p_max_pu"], other=0.0, inplace=True)
 
-    if solve_opts.get("load_shedding"):
+    load_shedding = solve_opts.get("load_shedding")
+    if load_shedding:
         # intersect between macroeconomic and surveybased willingness to pay
         # http://journal.frontiersin.org/article/10.3389/fenrg.2015.00055/full
         # TODO: retrieve color and nice name from config

From e550ba915dbfea1a8a4f48dc2efa18ca55741ad5 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Mon, 24 Apr 2023 14:08:25 +0200
Subject: [PATCH 033/176] Update add_extra_components.py

Note:
- battery charger and battery discharger has no nice name nor plot_color
- H2 electrolysis is similar to H2 Electrolysis in the tech_color
- H2 fuel cell is similar to H2 Fuel cell in the tech_color
- H2 pipeline do have have nice name and tech_color, but the addition of it depends on the link being true in config
---
 scripts/add_extra_components.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py
index 020370e5..0d3fcc6f 100644
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@@ -119,6 +119,8 @@ def attach_stores(n, costs, elec_opts):
             e_cyclic=True,
             capital_cost=costs.at["hydrogen storage underground", "capital_cost"],
         )
+        
+        n.madd("Carrier",["H2 electrolysis","H2 fuel cell"])
 
         n.madd(
             "Link",
@@ -161,6 +163,8 @@ def attach_stores(n, costs, elec_opts):
             capital_cost=costs.at["battery storage", "capital_cost"],
             marginal_cost=costs.at["battery", "marginal_cost"],
         )
+        
+        n.madd("Carrier",["battery charger","battery discharger"])
 
         n.madd(
             "Link",
@@ -213,6 +217,8 @@ def attach_hydrogen_pipelines(n, costs, elec_opts):
     h2_links.index = h2_links.apply(lambda c: f"H2 pipeline {c.bus0}-{c.bus1}", axis=1)
 
     # add pipelines
+    n.add("Carrier","H2 pipeline")
+    
     n.madd(
         "Link",
         h2_links.index,

From 12b6e8fb9f0a2da58b2f55cf486d210eb8a14a81 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 24 Apr 2023 22:08:36 +0000
Subject: [PATCH 034/176] [pre-commit.ci] pre-commit autoupdate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

updates:
- [github.com/PyCQA/docformatter: v1.6.1-rc1 → v1.6.3](https://github.com/PyCQA/docformatter/compare/v1.6.1-rc1...v1.6.3)
---
 .pre-commit-config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 4279b96a..03dffc2a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -39,7 +39,7 @@ repos:
 
   # Make docstrings PEP 257 compliant
 - repo: https://github.com/PyCQA/docformatter
-  rev: v1.6.1-rc1
+  rev: v1.6.3
   hooks:
   - id: docformatter
     args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"]

From 3625d401c619e5f6cf86efcc8cf1263f0a7a1ffe Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 24 Apr 2023 22:09:06 +0000
Subject: [PATCH 035/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/build_bus_regions.py  | 7 ++++---
 scripts/build_cop_profiles.py | 2 --
 scripts/build_gas_network.py  | 2 --
 3 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py
index ee90eef6..6dc3b5a4 100644
--- a/scripts/build_bus_regions.py
+++ b/scripts/build_bus_regions.py
@@ -56,9 +56,10 @@ logger = logging.getLogger(__name__)
 
 def voronoi_partition_pts(points, outline):
     """
-    Compute the polygons of a voronoi partition of `points` within the
-    polygon `outline`. Taken from
-    https://github.com/FRESNA/vresutils/blob/master/vresutils/graph.py
+    Compute the polygons of a voronoi partition of `points` within the polygon
+    `outline`. Taken from
+    https://github.com/FRESNA/vresutils/blob/master/vresutils/graph.py.
+
     Attributes
     ----------
     points : Nx2 - ndarray[dtype=float]
diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py
index 342bd7e0..5d36cd5b 100644
--- a/scripts/build_cop_profiles.py
+++ b/scripts/build_cop_profiles.py
@@ -10,8 +10,6 @@ The COP is a function of the temperature difference between source and
 sink.
 
 The quadratic regression used is based on Staffell et al. (2012)
-
-
 https://doi.org/10.1039/C2EE22653G.
 """
 
diff --git a/scripts/build_gas_network.py b/scripts/build_gas_network.py
index f3021022..23f58caa 100644
--- a/scripts/build_gas_network.py
+++ b/scripts/build_gas_network.py
@@ -27,8 +27,6 @@ def diameter_to_capacity(pipe_diameter_mm):
     bar -> 21.7  GW CH4 pipe capacity (LHV)
 
     Based on p.15 of
-
-
     https://gasforclimate2050.eu/wp-content/uploads/2020/07/2020_European-Hydrogen-Backbone_Report.pdf
     """
     # slopes definitions

From 71c1a7fbefffa6e5efdff8c8da8361f5b8065f2c Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Tue, 25 Apr 2023 14:43:04 +0200
Subject: [PATCH 036/176] add the function add_missing_carrier

add_missing_carrier() are applied on python script where new components are added. The function runs ideally after all of the new components are added and before add_nice_carrier_names()
---
 scripts/add_electricity.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 1d32bce1..57a18a73 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -120,6 +120,11 @@ def calculate_annuity(n, r):
     else:
         return 1 / n
 
+def add_missing_carrier(n):
+    components = [n.buses, n.generators, n.lines, n.links, n.storage_units, n.stores]
+    for c in components:
+        missing_carrier = np.setdiff1d(c.carrier.unique(),n.carriers.index)
+        n.madd("Carrier",missing_carrier)
 
 def _add_missing_carriers_from_costs(n, costs, carriers):
     missing_carriers = pd.Index(carriers).difference(n.carriers.index)
@@ -832,7 +837,8 @@ if __name__ == "__main__":
         estimate_renewable_capacities(n, snakemake.config)
 
     update_p_nom_max(n)
-
+    
+    add_missing_carrier(n)
     add_nice_carrier_names(n, snakemake.config)
 
     n.meta = snakemake.config

From 9b631189c5c6d43b481dd5a0e06a332e88f51623 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Tue, 25 Apr 2023 16:34:17 +0200
Subject: [PATCH 037/176] retrieve.smk: rename retrieve_load_data ->
 retrieve_electricity_demand

---
 doc/release_notes.rst   | 4 +++-
 doc/retrieve.rst        | 4 ++--
 doc/tutorial.rst        | 2 +-
 doc/tutorial_sector.rst | 4 ++--
 rules/retrieve.smk      | 4 ++--
 5 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index f859646b..192d828c 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -15,7 +15,9 @@ Upcoming Release
 
 * Bugfix: Correct typo in the CPLEX solver configuration in ``config.default.yaml``.
 
-* Renamed script file from PyPSA-EUR ``build_load_data`` to ``build_electricity_demand``.
+* Renamed script file from PyPSA-EUR ``build_load_data`` to ``build_electricity_demand`` and ``retrieve_load_data`` to ``retrieve_electricity_demand``.
+
+
 
 PyPSA-Eur 0.8.0 (18th March 2023)
 =================================
diff --git a/doc/retrieve.rst b/doc/retrieve.rst
index e4f47dec..cc93c3d9 100644
--- a/doc/retrieve.rst
+++ b/doc/retrieve.rst
@@ -80,8 +80,8 @@ This rule, as a substitute for :mod:`build_natura_raster`, downloads an already
     For details see :mod:`build_natura_raster`.
 
 
-Rule ``retrieve_load_data``
-================================
+Rule ``retrieve_electricity_demand``
+====================================
 
 This rule downloads hourly electric load data for each country from the `OPSD platform <data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv>`_.
 
diff --git a/doc/tutorial.rst b/doc/tutorial.rst
index f396b82c..1b87cefe 100644
--- a/doc/tutorial.rst
+++ b/doc/tutorial.rst
@@ -156,7 +156,7 @@ This triggers a workflow of multiple preceding jobs that depend on each rule's i
         20[label = "retrieve_cost_data", color = "0.30 0.6 0.85", style="rounded"];
         21[label = "build_powerplants", color = "0.16 0.6 0.85", style="rounded"];
         22[label = "build_electricity_demand", color = "0.00 0.6 0.85", style="rounded"];
-        23[label = "retrieve_load_data", color = "0.34 0.6 0.85", style="rounded,dashed"];
+        23[label = "retrieve_electricity_demand", color = "0.34 0.6 0.85", style="rounded,dashed"];
         1 -> 0
         2 -> 1
         20 -> 1
diff --git a/doc/tutorial_sector.rst b/doc/tutorial_sector.rst
index 039271db..29971e3a 100644
--- a/doc/tutorial_sector.rst
+++ b/doc/tutorial_sector.rst
@@ -141,7 +141,7 @@ successfully.
         19[label = "retrieve_cost_data", color = "0.50 0.6 0.85", style="rounded"];
         20[label = "build_powerplants", color = "0.49 0.6 0.85", style="rounded"];
         21[label = "build_electricity_demand", color = "0.39 0.6 0.85", style="rounded"];
-        22[label = "retrieve_load_data", color = "0.05 0.6 0.85", style="rounded"];
+        22[label = "retrieve_electricity_demand", color = "0.05 0.6 0.85", style="rounded"];
         23[label = "build_gas_input_locations", color = "0.45 0.6 0.85", style="rounded"];
         24[label = "prepare_network", color = "0.31 0.6 0.85", style="rounded"];
         25[label = "add_extra_components", color = "0.23 0.6 0.85", style="rounded"];
@@ -368,7 +368,7 @@ implemented in the workflow:
         19[label = "retrieve_cost_data", color = "0.04 0.6 0.85", style="rounded"];
         20[label = "build_powerplants", color = "0.28 0.6 0.85", style="rounded"];
         21[label = "build_electricity_demand", color = "0.46 0.6 0.85", style="rounded"];
-        22[label = "retrieve_load_data", color = "0.44 0.6 0.85", style="rounded"];
+        22[label = "retrieve_electricity_demand", color = "0.44 0.6 0.85", style="rounded"];
         23[label = "build_energy_totals", color = "0.53 0.6 0.85", style="rounded"];
         24[label = "build_population_weighted_energy_totals", color = "0.03 0.6 0.85", style="rounded"];
         25[label = "build_clustered_population_layouts", color = "0.34 0.6 0.85", style="rounded"];
diff --git a/rules/retrieve.smk b/rules/retrieve.smk
index 37deb44d..0a96406a 100644
--- a/rules/retrieve.smk
+++ b/rules/retrieve.smk
@@ -140,7 +140,7 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
             "../scripts/retrieve_gas_infrastructure_data.py"
 
 
-rule retrieve_load_data:
+rule retrieve_electricity_demand:
     input:
         HTTP.remote(
             "data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv",
@@ -150,7 +150,7 @@ rule retrieve_load_data:
     output:
         "data/load_raw.csv",
     log:
-        LOGS + "retrieve_load_data.log",
+        LOGS + "retrieve_electricity_demand.log",
     resources:
         mem_mb=5000,
     retries: 2

From 56b6718ba6b103062eaee0ff12680a88d9a7660d Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Tue, 25 Apr 2023 17:04:28 +0200
Subject: [PATCH 038/176] Update add_existing_baseyear.py

---
 scripts/add_existing_baseyear.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py
index 5bc0960d..1cc84a36 100644
--- a/scripts/add_existing_baseyear.py
+++ b/scripts/add_existing_baseyear.py
@@ -23,6 +23,7 @@ import pypsa
 import xarray as xr
 from _helpers import override_component_attrs, update_config_with_sector_opts
 from prepare_sector_network import cluster_heat_buses, define_spatial, prepare_costs
+from add_electricity import add_missing_carrier
 
 cc = coco.CountryConverter()
 
@@ -666,5 +667,7 @@ if __name__ == "__main__":
         cluster_heat_buses(n)
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
+    
+    add_missing_carrier(n)
 
     n.export_to_netcdf(snakemake.output[0])

From d9f9d4853a27d1f3e677dff16fb78ee94dced899 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Tue, 25 Apr 2023 17:06:17 +0200
Subject: [PATCH 039/176] Update add_extra_components.py

---
 scripts/add_extra_components.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py
index 0d3fcc6f..dcd000cd 100644
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@@ -57,6 +57,7 @@ import pandas as pd
 import pypsa
 from _helpers import configure_logging
 from add_electricity import (
+    add_missing_carrier,
     _add_missing_carriers_from_costs,
     add_nice_carrier_names,
     load_costs,
@@ -251,6 +252,8 @@ if __name__ == "__main__":
     attach_storageunits(n, costs, elec_config)
     attach_stores(n, costs, elec_config)
     attach_hydrogen_pipelines(n, costs, elec_config)
+    
+    add_missing_carrier(n)
 
     add_nice_carrier_names(n, snakemake.config)
 

From 756a814c636d8b402e53da74be48af7bb241eb75 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Tue, 25 Apr 2023 17:17:32 +0200
Subject: [PATCH 040/176] Update prepare_sector_network.py

---
 scripts/prepare_sector_network.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 6129d3b5..c0def274 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -29,6 +29,7 @@ from pypsa.geo import haversine_pts
 from pypsa.io import import_components_from_dataframe
 from scipy.stats import beta
 from vresutils.costdata import annuity
+from add_electricity import add_missing_carrier
 
 logger = logging.getLogger(__name__)
 
@@ -3398,5 +3399,7 @@ if __name__ == "__main__":
         cluster_heat_buses(n)
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
+    
+    add_missing_carrier(n)
 
     n.export_to_netcdf(snakemake.output[0])

From 74e9d56adb1ea4c71fdaa764efbb9e69a363e8af Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 25 Apr 2023 15:26:23 +0000
Subject: [PATCH 041/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/add_electricity.py        |  8 +++++---
 scripts/add_existing_baseyear.py  |  4 ++--
 scripts/add_extra_components.py   | 16 ++++++++--------
 scripts/base_network.py           |  2 +-
 scripts/prepare_sector_network.py |  4 ++--
 5 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 57a18a73..aa201c84 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -120,11 +120,13 @@ def calculate_annuity(n, r):
     else:
         return 1 / n
 
+
 def add_missing_carrier(n):
     components = [n.buses, n.generators, n.lines, n.links, n.storage_units, n.stores]
     for c in components:
-        missing_carrier = np.setdiff1d(c.carrier.unique(),n.carriers.index)
-        n.madd("Carrier",missing_carrier)
+        missing_carrier = np.setdiff1d(c.carrier.unique(), n.carriers.index)
+        n.madd("Carrier", missing_carrier)
+
 
 def _add_missing_carriers_from_costs(n, costs, carriers):
     missing_carriers = pd.Index(carriers).difference(n.carriers.index)
@@ -837,7 +839,7 @@ if __name__ == "__main__":
         estimate_renewable_capacities(n, snakemake.config)
 
     update_p_nom_max(n)
-    
+
     add_missing_carrier(n)
     add_nice_carrier_names(n, snakemake.config)
 
diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py
index 1cc84a36..a8a07322 100644
--- a/scripts/add_existing_baseyear.py
+++ b/scripts/add_existing_baseyear.py
@@ -22,8 +22,8 @@ import numpy as np
 import pypsa
 import xarray as xr
 from _helpers import override_component_attrs, update_config_with_sector_opts
-from prepare_sector_network import cluster_heat_buses, define_spatial, prepare_costs
 from add_electricity import add_missing_carrier
+from prepare_sector_network import cluster_heat_buses, define_spatial, prepare_costs
 
 cc = coco.CountryConverter()
 
@@ -667,7 +667,7 @@ if __name__ == "__main__":
         cluster_heat_buses(n)
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
-    
+
     add_missing_carrier(n)
 
     n.export_to_netcdf(snakemake.output[0])
diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py
index dcd000cd..88606305 100644
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@@ -57,8 +57,8 @@ import pandas as pd
 import pypsa
 from _helpers import configure_logging
 from add_electricity import (
-    add_missing_carrier,
     _add_missing_carriers_from_costs,
+    add_missing_carrier,
     add_nice_carrier_names,
     load_costs,
 )
@@ -120,8 +120,8 @@ def attach_stores(n, costs, elec_opts):
             e_cyclic=True,
             capital_cost=costs.at["hydrogen storage underground", "capital_cost"],
         )
-        
-        n.madd("Carrier",["H2 electrolysis","H2 fuel cell"])
+
+        n.madd("Carrier", ["H2 electrolysis", "H2 fuel cell"])
 
         n.madd(
             "Link",
@@ -164,8 +164,8 @@ def attach_stores(n, costs, elec_opts):
             capital_cost=costs.at["battery storage", "capital_cost"],
             marginal_cost=costs.at["battery", "marginal_cost"],
         )
-        
-        n.madd("Carrier",["battery charger","battery discharger"])
+
+        n.madd("Carrier", ["battery charger", "battery discharger"])
 
         n.madd(
             "Link",
@@ -218,8 +218,8 @@ def attach_hydrogen_pipelines(n, costs, elec_opts):
     h2_links.index = h2_links.apply(lambda c: f"H2 pipeline {c.bus0}-{c.bus1}", axis=1)
 
     # add pipelines
-    n.add("Carrier","H2 pipeline")
-    
+    n.add("Carrier", "H2 pipeline")
+
     n.madd(
         "Link",
         h2_links.index,
@@ -252,7 +252,7 @@ if __name__ == "__main__":
     attach_storageunits(n, costs, elec_config)
     attach_stores(n, costs, elec_config)
     attach_hydrogen_pipelines(n, costs, elec_config)
-    
+
     add_missing_carrier(n)
 
     add_nice_carrier_names(n, snakemake.config)
diff --git a/scripts/base_network.py b/scripts/base_network.py
index ecf21e1f..87504ce7 100644
--- a/scripts/base_network.py
+++ b/scripts/base_network.py
@@ -714,7 +714,7 @@ def base_network(
     n.name = "PyPSA-Eur"
 
     n.set_snapshots(pd.date_range(freq="h", **config["snapshots"]))
-    n.madd("Carrier",["AC","DC"])
+    n.madd("Carrier", ["AC", "DC"])
 
     n.import_components_from_dataframe(buses, "Bus")
     n.import_components_from_dataframe(lines, "Line")
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index c0def274..a0d2f472 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -22,6 +22,7 @@ from _helpers import (
     override_component_attrs,
     update_config_with_sector_opts,
 )
+from add_electricity import add_missing_carrier
 from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2
 from networkx.algorithms import complement
 from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation
@@ -29,7 +30,6 @@ from pypsa.geo import haversine_pts
 from pypsa.io import import_components_from_dataframe
 from scipy.stats import beta
 from vresutils.costdata import annuity
-from add_electricity import add_missing_carrier
 
 logger = logging.getLogger(__name__)
 
@@ -3399,7 +3399,7 @@ if __name__ == "__main__":
         cluster_heat_buses(n)
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
-    
+
     add_missing_carrier(n)
 
     n.export_to_netcdf(snakemake.output[0])

From df38edab0a06bf10d949ac14a1183648dfcdfe07 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Wed, 26 Apr 2023 04:28:36 +0200
Subject: [PATCH 042/176] fix bug in prepare_network.py

---
 scripts/prepare_network.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index 14a003ae..951cc6c9 100755
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -298,6 +298,8 @@ if __name__ == "__main__":
             break
 
     for o in opts:
+        if "+" not in o:
+            continue
         oo = o.split("+")
         suptechs = map(lambda c: c.split("-", 2)[0], n.carriers.index)
         if oo[0].startswith(tuple(suptechs)):

From a2e3e9163094c5ee2c712a82e98ceb972cbcf893 Mon Sep 17 00:00:00 2001
From: lisazeyen <35347358+lisazeyen@users.noreply.github.com>
Date: Wed, 26 Apr 2023 18:05:56 +0200
Subject: [PATCH 043/176] fix bug in load shedding constraint

---
 scripts/solve_network.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index fcdc4ed9..ff1c0ccf 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -166,7 +166,7 @@ def prepare_network(n, solve_opts=None, config=None):
             "Generator",
             buses_i,
             " load",
-            bus=n.buses.index,
+            bus=buses_i,
             carrier="load",
             sign=1e-3,  # Adjust sign to measure p and p_nom in kW instead of MW
             marginal_cost=load_shedding,  # Eur/kWh

From e24ca89a05fdb3d33578e3a4e4c0ef3e8e8411fd Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Thu, 27 Apr 2023 17:29:20 +0200
Subject: [PATCH 044/176] Update scripts/add_electricity.py

Co-authored-by: Fabian Hofmann <fab.hof@gmx.de>
---
 scripts/add_electricity.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index aa201c84..348e49af 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -125,7 +125,8 @@ def add_missing_carrier(n):
     components = [n.buses, n.generators, n.lines, n.links, n.storage_units, n.stores]
     for c in components:
         missing_carrier = np.setdiff1d(c.carrier.unique(), n.carriers.index)
-        n.madd("Carrier", missing_carrier)
+        if len(missing_carrier):
+            n.madd("Carrier", missing_carrier)
 
 
 def _add_missing_carriers_from_costs(n, costs, carriers):

From 72b75fd9feeeb137d3ad671419a8e6a574d2a041 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Fri, 28 Apr 2023 03:43:20 +0200
Subject: [PATCH 045/176] merge add_nice_carrier_names with add_missing_carrier
 to become add_missing_carrier_with_nice_names(n, config)

---
 scripts/add_electricity.py        | 33 ++++++++++++++-----------------
 scripts/add_existing_baseyear.py  |  4 ++--
 scripts/add_extra_components.py   |  7 ++-----
 scripts/prepare_sector_network.py |  4 ++--
 4 files changed, 21 insertions(+), 27 deletions(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 348e49af..456cc48c 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -121,13 +121,26 @@ def calculate_annuity(n, r):
         return 1 / n
 
 
-def add_missing_carrier(n):
+def add_missing_carriers_with_nice_names(n,config):
     components = [n.buses, n.generators, n.lines, n.links, n.storage_units, n.stores]
     for c in components:
         missing_carrier = np.setdiff1d(c.carrier.unique(), n.carriers.index)
         if len(missing_carrier):
             n.madd("Carrier", missing_carrier)
 
+    carrier_i = n.carriers.index
+    nice_names = (
+        pd.Series(config["plotting"]["nice_names"])
+        .reindex(carrier_i)
+        .fillna(carrier_i.to_series().str.title())
+    )
+    n.carriers["nice_name"] = nice_names
+    colors = pd.Series(config["plotting"]["tech_colors"]).reindex(carrier_i)
+    if colors.isna().any():
+        missing_i = list(colors.index[colors.isna()])
+        logger.warning(f"tech_colors for carriers {missing_i} not defined in config.")
+    n.carriers["color"] = colors
+
 
 def _add_missing_carriers_from_costs(n, costs, carriers):
     missing_carriers = pd.Index(carriers).difference(n.carriers.index)
@@ -694,21 +707,6 @@ def estimate_renewable_capacities(n, config):
             )
 
 
-def add_nice_carrier_names(n, config):
-    carrier_i = n.carriers.index
-    nice_names = (
-        pd.Series(config["plotting"]["nice_names"])
-        .reindex(carrier_i)
-        .fillna(carrier_i.to_series().str.title())
-    )
-    n.carriers["nice_name"] = nice_names
-    colors = pd.Series(config["plotting"]["tech_colors"]).reindex(carrier_i)
-    if colors.isna().any():
-        missing_i = list(colors.index[colors.isna()])
-        logger.warning(f"tech_colors for carriers {missing_i} not defined in config.")
-    n.carriers["color"] = colors
-
-
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
@@ -841,8 +839,7 @@ if __name__ == "__main__":
 
     update_p_nom_max(n)
 
-    add_missing_carrier(n)
-    add_nice_carrier_names(n, snakemake.config)
+    add_missing_carriers_with_nice_names(n, snakemake.config)
 
     n.meta = snakemake.config
     n.export_to_netcdf(snakemake.output[0])
diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py
index a8a07322..1435e660 100644
--- a/scripts/add_existing_baseyear.py
+++ b/scripts/add_existing_baseyear.py
@@ -22,7 +22,7 @@ import numpy as np
 import pypsa
 import xarray as xr
 from _helpers import override_component_attrs, update_config_with_sector_opts
-from add_electricity import add_missing_carrier
+from add_electricity import add_missing_carriers_with_nice_names
 from prepare_sector_network import cluster_heat_buses, define_spatial, prepare_costs
 
 cc = coco.CountryConverter()
@@ -668,6 +668,6 @@ if __name__ == "__main__":
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
 
-    add_missing_carrier(n)
+    add_missing_carriers_with_nice_names(n, snakemake.config)
 
     n.export_to_netcdf(snakemake.output[0])
diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py
index 88606305..da47b7b7 100644
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@@ -58,8 +58,7 @@ import pypsa
 from _helpers import configure_logging
 from add_electricity import (
     _add_missing_carriers_from_costs,
-    add_missing_carrier,
-    add_nice_carrier_names,
+    add_missing_carriers_with_nice_names,
     load_costs,
 )
 
@@ -253,9 +252,7 @@ if __name__ == "__main__":
     attach_stores(n, costs, elec_config)
     attach_hydrogen_pipelines(n, costs, elec_config)
 
-    add_missing_carrier(n)
-
-    add_nice_carrier_names(n, snakemake.config)
+    add_missing_carriers_with_nice_names(n, snakemake.config)
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
     n.export_to_netcdf(snakemake.output[0])
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index a0d2f472..d952cb69 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -22,7 +22,7 @@ from _helpers import (
     override_component_attrs,
     update_config_with_sector_opts,
 )
-from add_electricity import add_missing_carrier
+from add_electricity import add_missing_carriers_with_nice_names
 from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2
 from networkx.algorithms import complement
 from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation
@@ -3400,6 +3400,6 @@ if __name__ == "__main__":
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
 
-    add_missing_carrier(n)
+    add_missing_carriers_with_nice_names(n, snakemake.config)
 
     n.export_to_netcdf(snakemake.output[0])

From 699a4bd2e8e09035e3b93e0d117ba6874e119a8e Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 28 Apr 2023 01:43:39 +0000
Subject: [PATCH 046/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/add_electricity.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 456cc48c..2d92405d 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -121,7 +121,7 @@ def calculate_annuity(n, r):
         return 1 / n
 
 
-def add_missing_carriers_with_nice_names(n,config):
+def add_missing_carriers_with_nice_names(n, config):
     components = [n.buses, n.generators, n.lines, n.links, n.storage_units, n.stores]
     for c in components:
         missing_carrier = np.setdiff1d(c.carrier.unique(), n.carriers.index)

From f3ca907fb0758831c49e3d01b5f8a39b61c8734b Mon Sep 17 00:00:00 2001
From: Laurent Drouet <ldrouet@pm.me>
Date: Fri, 28 Apr 2023 15:23:44 +0200
Subject: [PATCH 047/176] Update installation.rst

---
 doc/installation.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/installation.rst b/doc/installation.rst
index b6fd574e..01fdafeb 100644
--- a/doc/installation.rst
+++ b/doc/installation.rst
@@ -39,7 +39,7 @@ The environment can be installed and activated using
 
 .. code:: bash
 
-    .../pypsa-eur % mamba create -f envs/environment.yaml
+    .../pypsa-eur % mamba env create -f envs/environment.yaml
 
     .../pypsa-eur % mamba activate pypsa-eur
 

From 51e09f5e61828019cdc350bbd93081dcdbe57c1b Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 30 Apr 2023 10:43:49 +0200
Subject: [PATCH 048/176] Address review comments

---
 scripts/prepare_sector_network.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 90891ad5..0c7a816c 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -1084,7 +1084,8 @@ def add_storage_and_grids(n, costs):
         )
 
     if options["hydrogen_turbine"]:
-        logger.info("Adding hydrogen turbine for re-electrification.")
+        logger.info("Adding hydrogen turbine for re-electrification. Assuming OCGT technology costs.")
+        # TODO: perhaps replace with hydrogen-specific technology assumptions.
 
         n.madd(
             "Link",

From bdeab82b494d2b7edd1700b5339f7ce3cbe89fa6 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sun, 30 Apr 2023 08:44:05 +0000
Subject: [PATCH 049/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/prepare_sector_network.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 0c7a816c..1cb7146e 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -1084,7 +1084,9 @@ def add_storage_and_grids(n, costs):
         )
 
     if options["hydrogen_turbine"]:
-        logger.info("Adding hydrogen turbine for re-electrification. Assuming OCGT technology costs.")
+        logger.info(
+            "Adding hydrogen turbine for re-electrification. Assuming OCGT technology costs."
+        )
         # TODO: perhaps replace with hydrogen-specific technology assumptions.
 
         n.madd(

From b997adcde7c21bef9e1788e38065310027359ace Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 1 May 2023 22:14:07 +0000
Subject: [PATCH 050/176] [pre-commit.ci] pre-commit autoupdate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

updates:
- [github.com/PyCQA/docformatter: v1.6.3 → v1.6.4](https://github.com/PyCQA/docformatter/compare/v1.6.3...v1.6.4)
---
 .pre-commit-config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 03dffc2a..02471b57 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -39,7 +39,7 @@ repos:
 
   # Make docstrings PEP 257 compliant
 - repo: https://github.com/PyCQA/docformatter
-  rev: v1.6.3
+  rev: v1.6.4
   hooks:
   - id: docformatter
     args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"]

From f6a40d36969b24c49b64c912f46f7d33c16cfd99 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 1 May 2023 22:14:42 +0000
Subject: [PATCH 051/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/build_retro_cost.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py
index cd1dae8d..9dbfc375 100644
--- a/scripts/build_retro_cost.py
+++ b/scripts/build_retro_cost.py
@@ -513,7 +513,7 @@ def prepare_cost_retro(country_iso_dic):
 
 def prepare_temperature_data():
     """
-    returns the temperature dependent data for each country:
+    Returns the temperature dependent data for each country:
 
     d_heat             : length of heating season pd.Series(index=countries) [days/year]
                          on those days, daily average temperature is below
@@ -621,7 +621,7 @@ def calculate_costs(u_values, l, cost_retro, window_assumptions):
 
 def calculate_new_u(u_values, l, l_weight, window_assumptions, k=0.035):
     """
-    calculate U-values after building retrofitting, depending on the old
+    Calculate U-values after building retrofitting, depending on the old
     U-values (u_values). This is for simple insulation measuers, adding an
     additional layer of insulation.
 
@@ -682,7 +682,7 @@ def map_tabula_to_hotmaps(df_tabula, df_hotmaps, column_prefix):
 
 def get_solar_gains_per_year(window_area):
     """
-    returns solar heat gains during heating season in [kWh/a] depending on the
+    Returns solar heat gains during heating season in [kWh/a] depending on the
     window area [m^2] of the building, assuming a equal distributed window
     orientation (east, south, north, west)
     """
@@ -718,7 +718,7 @@ def map_to_lstrength(l_strength, df):
 
 def calculate_heat_losses(u_values, data_tabula, l_strength, temperature_factor):
     """
-    calculates total annual heat losses Q_ht for different insulation
+    Calculates total annual heat losses Q_ht for different insulation
     thicknesses (l_strength), depending on current insulation state (u_values),
     standard building topologies and air ventilation from TABULA (data_tabula)
     and the accumulated difference between internal and external temperature
@@ -840,7 +840,7 @@ def calculate_heat_losses(u_values, data_tabula, l_strength, temperature_factor)
 
 def calculate_heat_gains(data_tabula, heat_transfer_perm2, d_heat):
     """
-    calculates heat gains Q_gain [W/m^2], which consititure from gains by:
+    Calculates heat gains Q_gain [W/m^2], which consititure from gains by:
 
     (1) solar radiation (2) internal heat gains
     """
@@ -885,7 +885,7 @@ def calculate_space_heat_savings(
     u_values, data_tabula, l_strength, temperature_factor, d_heat
 ):
     """
-    calculates space heat savings (dE_space [per unit of unrefurbished state])
+    Calculates space heat savings (dE_space [per unit of unrefurbished state])
     through retrofitting of the thermal envelope by additional insulation
     material (l_strength[m])
     """

From f355f816c3d2452e6ffe8a6dd25154c5a2f9cc5d Mon Sep 17 00:00:00 2001
From: energyls <leon1.schumm@oth-regensburg.de>
Date: Tue, 2 May 2023 17:20:06 +0200
Subject: [PATCH 052/176] docs: add stackoverflow issue template

---
 .github/ISSUE_TEMPLATE/config.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
index d8c04382..b357ab5f 100644
--- a/.github/ISSUE_TEMPLATE/config.yml
+++ b/.github/ISSUE_TEMPLATE/config.yml
@@ -3,3 +3,6 @@ contact_links:
 - name: PyPSA Mailing List
   url: https://groups.google.com/forum/#!forum/pypsa
   about: Please ask and answer general usage questions here.
+- name: Stackoverflow
+  url: https://stackoverflow.com/questions/tagged/pypsa
+  about: Please ask and answer code-related questions here.
\ No newline at end of file

From 4f890799455b11ccd671ebe0785e32d2c7d34d76 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 2 May 2023 15:21:48 +0000
Subject: [PATCH 053/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .github/ISSUE_TEMPLATE/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
index b357ab5f..03659213 100644
--- a/.github/ISSUE_TEMPLATE/config.yml
+++ b/.github/ISSUE_TEMPLATE/config.yml
@@ -5,4 +5,4 @@ contact_links:
   about: Please ask and answer general usage questions here.
 - name: Stackoverflow
   url: https://stackoverflow.com/questions/tagged/pypsa
-  about: Please ask and answer code-related questions here.
\ No newline at end of file
+  about: Please ask and answer code-related questions here.

From 2a1cdd1d7820ec242081cbab7a98789b2711772d Mon Sep 17 00:00:00 2001
From: energyls <leon1.schumm@oth-regensburg.de>
Date: Tue, 2 May 2023 17:39:01 +0200
Subject: [PATCH 054/176] docs: add shield and support section to readme

---
 README.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/README.md b/README.md
index 2973fa33..02cc3163 100644
--- a/README.md
+++ b/README.md
@@ -11,6 +11,7 @@ SPDX-License-Identifier: CC-BY-4.0
 [![Zenodo PyPSA-Eur-Sec](https://zenodo.org/badge/DOI/10.5281/zenodo.3938042.svg)](https://doi.org/10.5281/zenodo.3938042)
 [![Snakemake](https://img.shields.io/badge/snakemake-≥5.0.0-brightgreen.svg?style=flat)](https://snakemake.readthedocs.io)
 [![REUSE status](https://api.reuse.software/badge/github.com/pypsa/pypsa-eur)](https://api.reuse.software/info/github.com/pypsa/pypsa-eur)
+[![Stack Exchange questions](https://img.shields.io/stackexchange/stackoverflow/t/pypsa)](https://stackoverflow.com/questions/tagged/pypsa)
 
 # PyPSA-Eur: A Sector-Coupled Open Optimisation Model of the European Energy System
 
@@ -90,6 +91,14 @@ to 50-200 nodes.
 
 Already-built versions of the model can be found in the accompanying [Zenodo
 repository](https://doi.org/10.5281/zenodo.3601881).
+
+# Contributing and Support
+We strongly welcome anyone interested in contributing to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on GitHub.
+-   In case of code-related **questions**, please post on [stack overflow](https://stackoverflow.com/questions/tagged/pypsa).
+-   For non-programming related and more general questions please refer to the [mailing list](https://groups.google.com/group/pypsa).
+-   To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.gg/JTdvaEBb).
+-   For **bugs and feature requests**, please use the [PyPSA-Eur Github Issues page](https://github.com/PyPSA/pypsa-eur/issues).
+
 # Licence
 
 The code in PyPSA-Eur is released as free software under the

From 42a9f40680b958c918f5436887a8c73cd2d7fdc6 Mon Sep 17 00:00:00 2001
From: energyls <leon1.schumm@oth-regensburg.de>
Date: Tue, 2 May 2023 17:49:26 +0200
Subject: [PATCH 055/176] docs: add shield to rst

---
 doc/index.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/index.rst b/doc/index.rst
index 521f080b..743444dd 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -31,7 +31,9 @@ PyPSA-Eur: A Sector-Coupled Open Optimisation Model of the European Energy Syste
     :target: https://api.reuse.software/info/github.com/pypsa/pypsa-eur
     :alt: REUSE
 
-|
+|.. image:: https://img.shields.io/stackexchange/stackoverflow/t/pypsa
+   :target: https://stackoverflow.com/questions/tagged/pypsa
+   :alt: Stackoverflow
 
 PyPSA-Eur is an open model dataset of the European energy system at the
 transmission network level that covers the full ENTSO-E area. It covers demand

From a6948ad39e44ecc4caaad8ccb8c3044ab1db7139 Mon Sep 17 00:00:00 2001
From: energyls <leon1.schumm@oth-regensburg.de>
Date: Wed, 3 May 2023 10:36:54 +0200
Subject: [PATCH 056/176] docs: add support.rst and add to TOC

---
 doc/index.rst   |  1 +
 doc/support.rst | 10 ++++++++++
 2 files changed, 11 insertions(+)
 create mode 100644 doc/support.rst

diff --git a/doc/index.rst b/doc/index.rst
index 743444dd..2d53cfb0 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -276,4 +276,5 @@ The included ``.nc`` files are PyPSA network files which can be imported with Py
    licenses
    limitations
    contributing
+   support
    publications
diff --git a/doc/support.rst b/doc/support.rst
new file mode 100644
index 00000000..1cd39e35
--- /dev/null
+++ b/doc/support.rst
@@ -0,0 +1,10 @@
+
+-------
+Support
+-------
+
+* In case of code-related **questions**, please post on `stack overflow <https://stackoverflow.com/questions/tagged/pypsa>`_.
+* For non-programming related and more general questions please refer to the `mailing list <https://groups.google.com/group/pypsa>`_.
+* To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.gg/JTdvaEBb).
+* For **bugs and feature requests**, please use the `issue tracker <https://github.com/PyPSA/pypsa-eur/issues>`_.
+* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github <https://github.com/PyPSA/PyPSA>`_. For further information on how to contribute, please refer to :ref:`contributing`.
\ No newline at end of file

From 93d79da903c10ff51ebd1db494165ca99aa4c5ae Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 3 May 2023 08:37:21 +0000
Subject: [PATCH 057/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 doc/support.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/support.rst b/doc/support.rst
index 1cd39e35..f0a2c83c 100644
--- a/doc/support.rst
+++ b/doc/support.rst
@@ -7,4 +7,4 @@ Support
 * For non-programming related and more general questions please refer to the `mailing list <https://groups.google.com/group/pypsa>`_.
 * To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.gg/JTdvaEBb).
 * For **bugs and feature requests**, please use the `issue tracker <https://github.com/PyPSA/pypsa-eur/issues>`_.
-* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github <https://github.com/PyPSA/PyPSA>`_. For further information on how to contribute, please refer to :ref:`contributing`.
\ No newline at end of file
+* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github <https://github.com/PyPSA/PyPSA>`_. For further information on how to contribute, please refer to :ref:`contributing`.

From d5b1a05e379e1ce123383687983ed29a8db167d7 Mon Sep 17 00:00:00 2001
From: energyls <leon1.schumm@oth-regensburg.de>
Date: Wed, 3 May 2023 10:40:58 +0200
Subject: [PATCH 058/176] docs: add licence information to support.rst

---
 doc/support.rst | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/doc/support.rst b/doc/support.rst
index f0a2c83c..36d1a2dd 100644
--- a/doc/support.rst
+++ b/doc/support.rst
@@ -1,7 +1,11 @@
+..
+  SPDX-FileCopyrightText: 2019-2023 The PyPSA-Eur Authors
 
--------
+  SPDX-License-Identifier: CC-BY-4.0
+
+#######################
 Support
--------
+#######################
 
 * In case of code-related **questions**, please post on `stack overflow <https://stackoverflow.com/questions/tagged/pypsa>`_.
 * For non-programming related and more general questions please refer to the `mailing list <https://groups.google.com/group/pypsa>`_.

From e9787e77e4ce9bad1206de2495e5e5d2023fe7b7 Mon Sep 17 00:00:00 2001
From: energyls <leon1.schumm@oth-regensburg.de>
Date: Wed, 3 May 2023 10:53:47 +0200
Subject: [PATCH 059/176] docs: remove hyperref to check CI

---
 doc/support.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/support.rst b/doc/support.rst
index 36d1a2dd..395c5f31 100644
--- a/doc/support.rst
+++ b/doc/support.rst
@@ -11,4 +11,4 @@ Support
 * For non-programming related and more general questions please refer to the `mailing list <https://groups.google.com/group/pypsa>`_.
 * To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.gg/JTdvaEBb).
 * For **bugs and feature requests**, please use the `issue tracker <https://github.com/PyPSA/pypsa-eur/issues>`_.
-* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github <https://github.com/PyPSA/PyPSA>`_. For further information on how to contribute, please refer to :ref:`contributing`.
+* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github <https://github.com/PyPSA/PyPSA>`_. For further information on how to contribute, please refer to ...
\ No newline at end of file

From 817e82a80529dd34254643b9786a806eb2b6769c Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 3 May 2023 08:54:47 +0000
Subject: [PATCH 060/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 doc/support.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/support.rst b/doc/support.rst
index 395c5f31..c0b4981b 100644
--- a/doc/support.rst
+++ b/doc/support.rst
@@ -11,4 +11,4 @@ Support
 * For non-programming related and more general questions please refer to the `mailing list <https://groups.google.com/group/pypsa>`_.
 * To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.gg/JTdvaEBb).
 * For **bugs and feature requests**, please use the `issue tracker <https://github.com/PyPSA/pypsa-eur/issues>`_.
-* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github <https://github.com/PyPSA/PyPSA>`_. For further information on how to contribute, please refer to ...
\ No newline at end of file
+* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github <https://github.com/PyPSA/PyPSA>`_. For further information on how to contribute, please refer to ...

From 673aab2925931ecc030a9d55b42c233774d45d86 Mon Sep 17 00:00:00 2001
From: energyls <leon1.schumm@oth-regensburg.de>
Date: Wed, 3 May 2023 10:58:46 +0200
Subject: [PATCH 061/176] fix: fix typo in index.rst

---
 doc/index.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/index.rst b/doc/index.rst
index 2d53cfb0..314900f7 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -31,7 +31,7 @@ PyPSA-Eur: A Sector-Coupled Open Optimisation Model of the European Energy Syste
     :target: https://api.reuse.software/info/github.com/pypsa/pypsa-eur
     :alt: REUSE
 
-|.. image:: https://img.shields.io/stackexchange/stackoverflow/t/pypsa
+.. image:: https://img.shields.io/stackexchange/stackoverflow/t/pypsa
    :target: https://stackoverflow.com/questions/tagged/pypsa
    :alt: Stackoverflow
 

From da022db624b3d4fb1290ee4121ffee04c1a2a6dc Mon Sep 17 00:00:00 2001
From: energyls <leon1.schumm@oth-regensburg.de>
Date: Wed, 3 May 2023 11:03:54 +0200
Subject: [PATCH 062/176] docs: add reference to contributing

---
 doc/support.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/support.rst b/doc/support.rst
index c0b4981b..f57e2289 100644
--- a/doc/support.rst
+++ b/doc/support.rst
@@ -11,4 +11,4 @@ Support
 * For non-programming related and more general questions please refer to the `mailing list <https://groups.google.com/group/pypsa>`_.
 * To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.gg/JTdvaEBb).
 * For **bugs and feature requests**, please use the `issue tracker <https://github.com/PyPSA/pypsa-eur/issues>`_.
-* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github <https://github.com/PyPSA/PyPSA>`_. For further information on how to contribute, please refer to ...
+* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github <https://github.com/PyPSA/PyPSA>`_. For further information on how to contribute, please refer to :ref:`contributing`.
\ No newline at end of file

From f0453c42c0266468bc3cc0dfe1bf527f778dff2d Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 3 May 2023 09:05:26 +0000
Subject: [PATCH 063/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 doc/support.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/support.rst b/doc/support.rst
index f57e2289..36d1a2dd 100644
--- a/doc/support.rst
+++ b/doc/support.rst
@@ -11,4 +11,4 @@ Support
 * For non-programming related and more general questions please refer to the `mailing list <https://groups.google.com/group/pypsa>`_.
 * To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.gg/JTdvaEBb).
 * For **bugs and feature requests**, please use the `issue tracker <https://github.com/PyPSA/pypsa-eur/issues>`_.
-* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github <https://github.com/PyPSA/PyPSA>`_. For further information on how to contribute, please refer to :ref:`contributing`.
\ No newline at end of file
+* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github <https://github.com/PyPSA/PyPSA>`_. For further information on how to contribute, please refer to :ref:`contributing`.

From 4e8bbd67b4d1026c60a7f2befa0d4b0b651d96ed Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Wed, 3 May 2023 13:24:57 +0200
Subject: [PATCH 064/176] use sanitize_carriers to harmonize carrier
 adjustments

---
 scripts/add_electricity.py          | 104 +++++++++++++++++-----------
 scripts/add_existing_baseyear.py    |   4 +-
 scripts/add_extra_components.py     |  14 ++--
 scripts/build_renewable_profiles.py |   7 +-
 scripts/prepare_sector_network.py   |   4 +-
 5 files changed, 78 insertions(+), 55 deletions(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 2d92405d..98d46c82 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -121,12 +121,38 @@ def calculate_annuity(n, r):
         return 1 / n
 
 
-def add_missing_carriers_with_nice_names(n, config):
-    components = [n.buses, n.generators, n.lines, n.links, n.storage_units, n.stores]
-    for c in components:
-        missing_carrier = np.setdiff1d(c.carrier.unique(), n.carriers.index)
-        if len(missing_carrier):
-            n.madd("Carrier", missing_carrier)
+def sanitize_carriers(n, config):
+    """
+    Sanitize the carrier information in a PyPSA Network object.
+
+    The function ensures that all unique carrier names are present in the network's
+    carriers attribute, and adds nice names and colors for each carrier according
+    to the provided configuration dictionary.
+
+    Parameters
+    ----------
+    n : pypsa.Network
+        A PyPSA Network object that represents an electrical power system.
+    config : dict
+        A dictionary containing configuration information, specifically the
+        "plotting" key with "nice_names" and "tech_colors" keys for carriers.
+
+    Returns
+    -------
+    None
+        The function modifies the 'n' PyPSA Network object in-place, updating the
+        carriers attribute with nice names and colors.
+
+    Warnings
+    --------
+    Raises a warning if any carrier's "tech_colors" are not defined in the config dictionary.
+    """
+
+    for c in n.iterate_components():
+        if "carrier" in c.df:
+            missing_carrier = set(c.df.carrier.unique()) - set(n.carriers.index) - {""}
+            if len(missing_carrier):
+                n.madd("Carrier", missing_carrier)
 
     carrier_i = n.carriers.index
     nice_names = (
@@ -142,18 +168,12 @@ def add_missing_carriers_with_nice_names(n, config):
     n.carriers["color"] = colors
 
 
-def _add_missing_carriers_from_costs(n, costs, carriers):
-    missing_carriers = pd.Index(carriers).difference(n.carriers.index)
-    if missing_carriers.empty:
-        return
-
-    emissions_cols = (
-        costs.columns.to_series().loc[lambda s: s.str.endswith("_emissions")].values
-    )
-    suptechs = missing_carriers.str.split("-").str[0]
-    emissions = costs.loc[suptechs, emissions_cols].fillna(0.0)
-    emissions.index = missing_carriers
-    n.import_components_from_dataframe(emissions, "Carrier")
+def add_co2_emissions(n, costs, carriers):
+    """
+    Add CO2 emissions to the network's carriers attribute.
+    """
+    suptechs = n.carriers.loc[carriers].index.str.split("-").str[0]
+    n.carriers.loc[carriers, "co2_emissions"] = costs.co2_emissions[suptechs].values
 
 
 def load_costs(tech_costs, config, elec_config, Nyears=1.0):
@@ -316,57 +336,56 @@ def update_transmission_costs(n, costs, length_factor=1.0):
 
 
 def attach_wind_and_solar(
-    n, costs, input_profiles, technologies, extendable_carriers, line_length_factor=1
+    n, costs, input_profiles, carriers, extendable_carriers, line_length_factor=1
 ):
-    # TODO: rename tech -> carrier, technologies -> carriers
-    _add_missing_carriers_from_costs(n, costs, technologies)
+    n.madd("Carrier", carriers)
 
-    for tech in technologies:
-        if tech == "hydro":
+    for car in carriers:
+        if car == "hydro":
             continue
 
-        with xr.open_dataset(getattr(input_profiles, "profile_" + tech)) as ds:
+        with xr.open_dataset(getattr(input_profiles, "profile_" + car)) as ds:
             if ds.indexes["bus"].empty:
                 continue
 
-            suptech = tech.split("-", 2)[0]
-            if suptech == "offwind":
+            supcar = car.split("-", 2)[0]
+            if supcar == "offwind":
                 underwater_fraction = ds["underwater_fraction"].to_pandas()
                 connection_cost = (
                     line_length_factor
                     * ds["average_distance"].to_pandas()
                     * (
                         underwater_fraction
-                        * costs.at[tech + "-connection-submarine", "capital_cost"]
+                        * costs.at[car + "-connection-submarine", "capital_cost"]
                         + (1.0 - underwater_fraction)
-                        * costs.at[tech + "-connection-underground", "capital_cost"]
+                        * costs.at[car + "-connection-underground", "capital_cost"]
                     )
                 )
                 capital_cost = (
                     costs.at["offwind", "capital_cost"]
-                    + costs.at[tech + "-station", "capital_cost"]
+                    + costs.at[car + "-station", "capital_cost"]
                     + connection_cost
                 )
                 logger.info(
                     "Added connection cost of {:0.0f}-{:0.0f} Eur/MW/a to {}".format(
-                        connection_cost.min(), connection_cost.max(), tech
+                        connection_cost.min(), connection_cost.max(), car
                     )
                 )
             else:
-                capital_cost = costs.at[tech, "capital_cost"]
+                capital_cost = costs.at[car, "capital_cost"]
 
             n.madd(
                 "Generator",
                 ds.indexes["bus"],
-                " " + tech,
+                " " + car,
                 bus=ds.indexes["bus"],
-                carrier=tech,
-                p_nom_extendable=tech in extendable_carriers["Generator"],
+                carrier=car,
+                p_nom_extendable=car in extendable_carriers["Generator"],
                 p_nom_max=ds["p_nom_max"].to_pandas(),
                 weight=ds["weight"].to_pandas(),
-                marginal_cost=costs.at[suptech, "marginal_cost"],
+                marginal_cost=costs.at[supcar, "marginal_cost"],
                 capital_cost=capital_cost,
-                efficiency=costs.at[suptech, "efficiency"],
+                efficiency=costs.at[supcar, "efficiency"],
                 p_max_pu=ds["profile"].transpose("time", "bus").to_pandas(),
             )
 
@@ -380,8 +399,9 @@ def attach_conventional_generators(
     conventional_config,
     conventional_inputs,
 ):
-    carriers = set(conventional_carriers) | set(extendable_carriers["Generator"])
-    _add_missing_carriers_from_costs(n, costs, carriers)
+    carriers = list(set(conventional_carriers) | set(extendable_carriers["Generator"]))
+    n.madd("Carrier", carriers)
+    add_co2_emissions(n, costs, carriers)
 
     ppl = (
         ppl.query("carrier in @carriers")
@@ -435,7 +455,8 @@ def attach_conventional_generators(
 
 
 def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **config):
-    _add_missing_carriers_from_costs(n, costs, carriers)
+    n.madd("Carrier", carriers)
+    add_co2_emissions(n, costs, carriers)
 
     ppl = (
         ppl.query('carrier == "hydro"')
@@ -567,7 +588,8 @@ def attach_extendable_generators(n, costs, ppl, carriers):
     logger.warning(
         "The function `attach_extendable_generators` is deprecated in v0.5.0."
     )
-    _add_missing_carriers_from_costs(n, costs, carriers)
+    n.madd("Carrier", carriers)
+    add_co2_emissions(n, costs, carriers)
 
     for tech in carriers:
         if tech.startswith("OCGT"):
@@ -839,7 +861,7 @@ if __name__ == "__main__":
 
     update_p_nom_max(n)
 
-    add_missing_carriers_with_nice_names(n, snakemake.config)
+    sanitize_carriers(n, snakemake.config)
 
     n.meta = snakemake.config
     n.export_to_netcdf(snakemake.output[0])
diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py
index 1435e660..6999a16b 100644
--- a/scripts/add_existing_baseyear.py
+++ b/scripts/add_existing_baseyear.py
@@ -22,7 +22,7 @@ import numpy as np
 import pypsa
 import xarray as xr
 from _helpers import override_component_attrs, update_config_with_sector_opts
-from add_electricity import add_missing_carriers_with_nice_names
+from add_electricity import sanitize_carriers
 from prepare_sector_network import cluster_heat_buses, define_spatial, prepare_costs
 
 cc = coco.CountryConverter()
@@ -668,6 +668,6 @@ if __name__ == "__main__":
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
 
-    add_missing_carriers_with_nice_names(n, snakemake.config)
+    sanitize_carriers(n, snakemake.config)
 
     n.export_to_netcdf(snakemake.output[0])
diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py
index da47b7b7..e3cc12ec 100644
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@@ -56,11 +56,7 @@ import numpy as np
 import pandas as pd
 import pypsa
 from _helpers import configure_logging
-from add_electricity import (
-    _add_missing_carriers_from_costs,
-    add_missing_carriers_with_nice_names,
-    load_costs,
-)
+from add_electricity import load_costs, sanitize_carriers
 
 idx = pd.IndexSlice
 
@@ -71,7 +67,7 @@ def attach_storageunits(n, costs, elec_opts):
     carriers = elec_opts["extendable_carriers"]["StorageUnit"]
     max_hours = elec_opts["max_hours"]
 
-    _add_missing_carriers_from_costs(n, costs, carriers)
+    n.madd("Carrier", carriers)
 
     buses_i = n.buses.index
 
@@ -102,7 +98,7 @@ def attach_storageunits(n, costs, elec_opts):
 def attach_stores(n, costs, elec_opts):
     carriers = elec_opts["extendable_carriers"]["Store"]
 
-    _add_missing_carriers_from_costs(n, costs, carriers)
+    n.madd("Carrier", carriers)
 
     buses_i = n.buses.index
     bus_sub_dict = {k: n.buses[k].values for k in ["x", "y", "country"]}
@@ -204,6 +200,8 @@ def attach_hydrogen_pipelines(n, costs, elec_opts):
         "`config.yaml` at `electricity: extendable_carriers: Store:`."
     )
 
+    n.add("Carrier", "H2 pipeline")
+
     # determine bus pairs
     attrs = ["bus0", "bus1", "length"]
     candidates = pd.concat(
@@ -252,7 +250,7 @@ if __name__ == "__main__":
     attach_stores(n, costs, elec_config)
     attach_hydrogen_pipelines(n, costs, elec_config)
 
-    add_missing_carriers_with_nice_names(n, snakemake.config)
+    sanitize_carriers(n, snakemake.config)
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
     n.export_to_netcdf(snakemake.output[0])
diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py
index c0288aee..086e4e39 100644
--- a/scripts/build_renewable_profiles.py
+++ b/scripts/build_renewable_profiles.py
@@ -216,8 +216,11 @@ if __name__ == "__main__":
     if correction_factor != 1.0:
         logger.info(f"correction_factor is set as {correction_factor}")
 
-    cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1)
-    client = Client(cluster, asynchronous=True)
+    if nprocesses > 1:
+        # cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1)
+        client = Client(n_workers=2, threads_per_worker=2, memory_limit="1GB")
+    else:
+        client = None
 
     cutout = atlite.Cutout(snakemake.input.cutout)
     regions = gpd.read_file(snakemake.input.regions)
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index d952cb69..bd82e46e 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -22,7 +22,7 @@ from _helpers import (
     override_component_attrs,
     update_config_with_sector_opts,
 )
-from add_electricity import add_missing_carriers_with_nice_names
+from add_electricity import sanitize_carriers
 from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2
 from networkx.algorithms import complement
 from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation
@@ -3400,6 +3400,6 @@ if __name__ == "__main__":
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
 
-    add_missing_carriers_with_nice_names(n, snakemake.config)
+    sanitize_carriers(n, snakemake.config)
 
     n.export_to_netcdf(snakemake.output[0])

From 13b4327e0163e58a1a3edf4933dfbdf9e60f5d10 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Wed, 3 May 2023 14:00:10 +0200
Subject: [PATCH 065/176] add_extra_components: revert carrier addition

---
 scripts/add_extra_components.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py
index e3cc12ec..926d3906 100644
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@@ -200,8 +200,6 @@ def attach_hydrogen_pipelines(n, costs, elec_opts):
         "`config.yaml` at `electricity: extendable_carriers: Store:`."
     )
 
-    n.add("Carrier", "H2 pipeline")
-
     # determine bus pairs
     attrs = ["bus0", "bus1", "length"]
     candidates = pd.concat(

From a28f2c18112fc902b708b539fbf9e11e4530b4af Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Fri, 5 May 2023 00:40:34 +0200
Subject: [PATCH 066/176] add tech_colors for previously missing carrier

---
 config/config.default.yaml | 60 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 58 insertions(+), 2 deletions(-)

diff --git a/config/config.default.yaml b/config/config.default.yaml
index 10e6a6ed..ae3ae655 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -790,6 +790,11 @@ plotting:
     solar: "#f9d002"
     solar PV: "#f9d002"
     solar thermal: '#ffbf2b'
+    residential rural solar thermal: '#f1c069'
+    services rural solar thermal: '#eabf61'
+    residential urban decentral solar thermal: '#e5bc5a'
+    services urban decentral solar thermal: '#dfb953'
+    urban central solar thermal: '#d7b24c'
     solar rooftop: '#ffea80'
     # gas
     OCGT: '#e0986c'
@@ -798,9 +803,15 @@ plotting:
     gas boiler: '#db6a25'
     gas boilers: '#db6a25'
     gas boiler marginal: '#db6a25'
+    residential rural gas boiler: '#d4722e'
+    residential urban decentral gas boiler: '#cb7a36'
+    services rural gas boiler: '#c4813f'
+    services urban decentral gas boiler: '#ba8947'
+    urban central gas boiler: '#b0904f'
     gas: '#e05b09'
     fossil gas: '#e05b09'
     natural gas: '#e05b09'
+    biogas to gas: '#e36311'
     CCGT: '#a85522'
     CCGT marginal: '#a85522'
     allam: '#B98F76'
@@ -838,13 +849,20 @@ plotting:
     solid biomass for industry CC: '#47411c'
     solid biomass for industry co2 from atmosphere: '#736412'
     solid biomass for industry co2 to stored: '#47411c'
+    urban central solid biomass CHP: '#9d9042'
+    urban central solid biomass CHP CC: '#6c5d28'
     biomass boiler: '#8A9A5B'
+    residential rural biomass boiler: '#a1a066'
+    residential urban decentral biomass boiler: '#b0b87b'
+    services rural biomass boiler: '#c6cf98'
+    services urban decentral biomass boiler: '#dde5b5'
     biomass to liquid: '#32CD32'
     BioSNG: '#123456'
     # power transmission
     lines: '#6c9459'
     transmission lines: '#6c9459'
     electricity distribution grid: '#97ad8c'
+    low voltage: '#97ad8c'
     # electricity demand
     Electric load: '#110d63'
     electric demand: '#110d63'
@@ -855,24 +873,48 @@ plotting:
     # battery + EVs
     battery: '#ace37f'
     battery storage: '#ace37f'
+    battery charger: '#88a75b'
+    battery discharger: '#5d4e29'
     home battery: '#80c944'
     home battery storage: '#80c944'
+    home battery charger: '#5e8032'
+    home battery discharger: '#3c5221'
     BEV charger: '#baf238'
     V2G: '#e5ffa8'
     land transport EV: '#baf238'
     Li ion: '#baf238'
     # hot water storage
     water tanks: '#e69487'
+    residential rural water tanks: '#f7b7a3'
+    services rural water tanks: '#f3afa3'
+    residential urban decentral water tanks: '#f2b2a3'
+    services urban decentral water tanks: '#f1b4a4'
+    urban central water tanks: '#e9977d'
     hot water storage: '#e69487'
-    hot water charging: '#e69487'
-    hot water discharging: '#e69487'
+    hot water charging: '#e8998b'
+    urban central water tanks charger: '#b57a67'
+    residential rural water tanks charger: '#b4887c'
+    residential urban decentral water tanks charger: '#b39995'
+    services rural water tanks charger: '#b3abb0'
+    services urban decentral water tanks charger: '#b3becc'
+    hot water discharging: '#e99c8e'
+    urban central water tanks discharger: '#b9816e'
+    residential rural water tanks discharger: '#ba9685'
+    residential urban decentral water tanks discharger: '#baac9e'
+    services rural water tanks discharger: '#bbc2b8'
+    services urban decentral water tanks discharger: '#bdd8d3'
     # heat demand
     Heat load: '#cc1f1f'
     heat: '#cc1f1f'
     heat demand: '#cc1f1f'
     rural heat: '#ff5c5c'
+    residential rural heat: '#ff7c7c'
+    services rural heat: '#ff9c9c'
     central heat: '#cc1f1f'
+    urban central heat: '#d15959'
     decentral heat: '#750606'
+    residential urban decentral heat: '#a33c3c'
+    services urban decentral heat: '#cc1f1f'
     low-temperature heat for industry: '#8f2727'
     process heat: '#ff0000'
     agriculture heat: '#d9a5a5'
@@ -880,14 +922,26 @@ plotting:
     heat pumps: '#2fb537'
     heat pump: '#2fb537'
     air heat pump: '#36eb41'
+    residential urban decentral air heat pump: '#48f74f'
+    services urban decentral air heat pump: '#5af95d'
+    urban central air heat pump: '#6cfb6b'
     ground heat pump: '#2fb537'
+    residential rural ground heat pump: '#48f74f'
+    services rural ground heat pump: '#5af95d'
     Ambient: '#98eb9d'
     CHP: '#8a5751'
+    urban central gas CHP: '#8d5e56'
     CHP CC: '#634643'
+    urban central gas CHP CC: '#6e4e4c'
     CHP heat: '#8a5751'
     CHP electric: '#8a5751'
     district heating: '#e8beac'
     resistive heater: '#d8f9b8'
+    residential rural resistive heater: '#bef5b5'
+    residential urban decentral resistive heater: '#b2f1a9'
+    services rural resistive heater: '#a5ed9d'
+    services urban decentral resistive heater: '#98e991'
+    urban central resistive heater: '#8cdf85'
     retrofitting: '#8487e8'
     building retrofitting: '#8487e8'
     # hydrogen
@@ -899,6 +953,7 @@ plotting:
     SMR CC: '#4f1745'
     H2 liquefaction: '#d647bd'
     hydrogen storage: '#bf13a0'
+    H2 Store: '#bf13a0'
     H2 storage: '#bf13a0'
     land transport fuel cell: '#6b3161'
     H2 pipeline: '#f081dc'
@@ -954,6 +1009,7 @@ plotting:
     waste: '#e3d37d'
     other: '#000000'
     geothermal: '#ba91b1'
+    AC: "#70af1d"
     AC-AC: "#70af1d"
     AC line: "#70af1d"
     links: "#8a1caf"

From 5603951946da59ce85e59e981e705858cf82b308 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Fri, 5 May 2023 11:32:16 +0200
Subject: [PATCH 067/176] fixed little things remaining

---
 config/config.default.yaml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/config/config.default.yaml b/config/config.default.yaml
index ae3ae655..1fe6adde 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -824,6 +824,11 @@ plotting:
     # oil
     oil: '#c9c9c9'
     oil boiler: '#adadad'
+    residential rural oil boiler: '#a9a9a9'
+    services rural oil boiler: '#a5a5a5'
+    residential urban decentral oil boiler: '#a1a1a1'
+    urban central oil boiler: '#9d9d9d'
+    services urban decentral oil boiler: '#999999'
     agriculture machinery oil: '#949494'
     shipping oil: "#808080"
     land transport oil: '#afafaf'
@@ -959,8 +964,10 @@ plotting:
     H2 pipeline: '#f081dc'
     H2 pipeline retrofitted: '#ba99b5'
     H2 Fuel Cell: '#c251ae'
+    H2 fuel cell: '#c251ae'
     H2 turbine: '#991f83'
     H2 Electrolysis: '#ff29d9'
+    H2 electrolysis: '#ff29d9'
     # ammonia
     NH3: '#46caf0'
     ammonia: '#46caf0'
@@ -1014,5 +1021,6 @@ plotting:
     AC line: "#70af1d"
     links: "#8a1caf"
     HVDC links: "#8a1caf"
+    DC: "#8a1caf"
     DC-DC: "#8a1caf"
     DC link: "#8a1caf"

From f6597f43a0417e6074da6125898a386eedcb9767 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Fri, 5 May 2023 12:16:16 +0200
Subject: [PATCH 068/176] Update release_notes.rst

---
 doc/release_notes.rst | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index 9be10a4a..e1550a93 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -17,11 +17,13 @@ Upcoming Release
 
 * Renamed script file from PyPSA-EUR ``build_load_data`` to ``build_electricity_demand`` and ``retrieve_load_data`` to ``retrieve_electricity_demand``.
 
-
-
 * Add plain hydrogen turbine as additional re-electrification option besides
   hydrogen fuel cell. Add switches for both re-electrification options under
   ``sector: hydrogen_turbine:`` and ``sector: hydrogen_fuel_cell:``.
+  
+* A new function named ``sanitize_carrier`` ensures that all unique carrier names are present in the network's carriers attribute, and adds nice names and colors for each carrier according to the provided configuration dictionary.
+
+* Additional tech_color are added to include previously unlisted carriers.
 
 PyPSA-Eur 0.8.0 (18th March 2023)
 =================================

From b5f5b35f41208af238d6f6bc2bdb2a4fd1f71110 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 5 May 2023 10:16:32 +0000
Subject: [PATCH 069/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 doc/release_notes.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index e1550a93..29aff48e 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -20,7 +20,7 @@ Upcoming Release
 * Add plain hydrogen turbine as additional re-electrification option besides
   hydrogen fuel cell. Add switches for both re-electrification options under
   ``sector: hydrogen_turbine:`` and ``sector: hydrogen_fuel_cell:``.
-  
+
 * A new function named ``sanitize_carrier`` ensures that all unique carrier names are present in the network's carriers attribute, and adds nice names and colors for each carrier according to the provided configuration dictionary.
 
 * Additional tech_color are added to include previously unlisted carriers.

From e9c5af27a31e2bded864035c7f8bfbaba48e0311 Mon Sep 17 00:00:00 2001
From: Max Parzen <max.parzen@ed.ac.uk>
Date: Sun, 7 May 2023 14:10:18 +0100
Subject: [PATCH 070/176] bump python version and add setuptools, myst-parser

---
 .readthedocs.yml     | 6 +++++-
 doc/requirements.txt | 2 ++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/.readthedocs.yml b/.readthedocs.yml
index 3d7a86b2..c74e12a2 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -4,8 +4,12 @@
 
 version: 2
 
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
 python:
-  version: 3.8
   install:
   - requirements: doc/requirements.txt
   system_packages: true
diff --git a/doc/requirements.txt b/doc/requirements.txt
index d5c71da9..2f08b8d9 100644
--- a/doc/requirements.txt
+++ b/doc/requirements.txt
@@ -2,9 +2,11 @@
 #
 # SPDX-License-Identifier: CC0-1.0
 
+setuptools
 sphinx
 sphinx_book_theme
 sphinxcontrib-bibtex
+myst-parser  # recommark is deprecated, https://stackoverflow.com/a/71660856/13573820
 
 pypsa
 vresutils>=0.3.1

From 510e544a4fd9350671eead4df41ae9d39abc9758 Mon Sep 17 00:00:00 2001
From: Max Parzen <max.parzen@ed.ac.uk>
Date: Sun, 7 May 2023 14:19:37 +0100
Subject: [PATCH 071/176] add release note

---
 doc/release_notes.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index 9be10a4a..8e6bfdd4 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -17,7 +17,7 @@ Upcoming Release
 
 * Renamed script file from PyPSA-EUR ``build_load_data`` to ``build_electricity_demand`` and ``retrieve_load_data`` to ``retrieve_electricity_demand``.
 
-
+* Fix docs readthedocs built
 
 * Add plain hydrogen turbine as additional re-electrification option besides
   hydrogen fuel cell. Add switches for both re-electrification options under

From 8a42ff77ad9e37f55eba2252e0e942902f5be08c Mon Sep 17 00:00:00 2001
From: Max Parzen <max.parzen@ed.ac.uk>
Date: Sun, 7 May 2023 14:24:58 +0100
Subject: [PATCH 072/176] add myst_parser to conf.py

---
 doc/conf.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/conf.py b/doc/conf.py
index 4b3978af..8111c86c 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -36,6 +36,7 @@ sys.path.insert(0, os.path.abspath("../scripts"))
 extensions = [
     #'sphinx.ext.autodoc',
     #'sphinx.ext.autosummary',
+    "myst_parser",
     "sphinx.ext.autosectionlabel",
     "sphinx.ext.intersphinx",
     "sphinx.ext.todo",

From e62d947fb850686440bb2f55df74d994a854e35d Mon Sep 17 00:00:00 2001
From: Max Parzen <max.parzen@ed.ac.uk>
Date: Sun, 7 May 2023 14:29:27 +0100
Subject: [PATCH 073/176] set system_packages false

---
 .readthedocs.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.readthedocs.yml b/.readthedocs.yml
index c74e12a2..35eff5de 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -12,4 +12,4 @@ build:
 python:
   install:
   - requirements: doc/requirements.txt
-  system_packages: true
+  system_packages: false

From 5bfa6237e8eaacad41ed9004dda86f4b1651afb4 Mon Sep 17 00:00:00 2001
From: Max Parzen <max.parzen@ed.ac.uk>
Date: Sun, 7 May 2023 15:44:33 +0100
Subject: [PATCH 074/176] add graphviz

---
 .readthedocs.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.readthedocs.yml b/.readthedocs.yml
index 35eff5de..b5dc5eba 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -8,8 +8,11 @@ build:
   os: ubuntu-22.04
   tools:
     python: "3.11"
+  apt_packages:
+    - graphviz
 
 python:
   install:
   - requirements: doc/requirements.txt
   system_packages: false
+  apt_packages:

From e18b188e81f64656b45232da8ec15c9954ef2281 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sun, 7 May 2023 14:44:54 +0000
Subject: [PATCH 075/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .readthedocs.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.readthedocs.yml b/.readthedocs.yml
index b5dc5eba..0ee32009 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -9,7 +9,7 @@ build:
   tools:
     python: "3.11"
   apt_packages:
-    - graphviz
+  - graphviz
 
 python:
   install:

From dad04a64a1db152813c11ec02dd30c4b03747e1d Mon Sep 17 00:00:00 2001
From: Max Parzen <max.parzen@ed.ac.uk>
Date: Sun, 7 May 2023 15:57:42 +0100
Subject: [PATCH 076/176] Update .readthedocs.yml

---
 .readthedocs.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.readthedocs.yml b/.readthedocs.yml
index 0ee32009..b5dc5eba 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -9,7 +9,7 @@ build:
   tools:
     python: "3.11"
   apt_packages:
-  - graphviz
+    - graphviz
 
 python:
   install:

From c431cf6f44a221dc6e7971d4977a1575c6d16867 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sun, 7 May 2023 14:57:56 +0000
Subject: [PATCH 077/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .readthedocs.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.readthedocs.yml b/.readthedocs.yml
index b5dc5eba..0ee32009 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -9,7 +9,7 @@ build:
   tools:
     python: "3.11"
   apt_packages:
-    - graphviz
+  - graphviz
 
 python:
   install:

From 702bb59a18ade47321440112ead6598bb0e4a0d2 Mon Sep 17 00:00:00 2001
From: Max Parzen <max.parzen@ed.ac.uk>
Date: Sun, 7 May 2023 16:06:33 +0100
Subject: [PATCH 078/176] fix artefact

---
 .readthedocs.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.readthedocs.yml b/.readthedocs.yml
index 0ee32009..900dba1e 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -15,4 +15,3 @@ python:
   install:
   - requirements: doc/requirements.txt
   system_packages: false
-  apt_packages:

From 8f7667f0f90048bafdff1f35d7ded7993eb68c83 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 8 May 2023 22:30:27 +0000
Subject: [PATCH 079/176] [pre-commit.ci] pre-commit autoupdate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

updates:
- [github.com/PyCQA/docformatter: v1.6.4 → v1.6.5](https://github.com/PyCQA/docformatter/compare/v1.6.4...v1.6.5)
---
 .pre-commit-config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 02471b57..88685f3b 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -39,7 +39,7 @@ repos:
 
   # Make docstrings PEP 257 compliant
 - repo: https://github.com/PyCQA/docformatter
-  rev: v1.6.4
+  rev: v1.6.5
   hooks:
   - id: docformatter
     args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"]

From 5c2e3c9fc136b12ea8953fada095d335541d22ca Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Wed, 10 May 2023 09:58:25 +0200
Subject: [PATCH 080/176] add Shape2Shapes in add_electricit.py

---
 scripts/add_electricity.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 1d32bce1..64cf442e 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -94,7 +94,9 @@ import pypsa
 import xarray as xr
 from _helpers import configure_logging, update_p_nom_max
 from powerplantmatching.export import map_country_bus
-from vresutils import transfer as vtransfer
+from itertools import product
+import scipy.sparse as sparse
+from shapely.prepared import prep
 
 idx = pd.IndexSlice
 
@@ -215,6 +217,20 @@ def load_powerplants(ppl_fn):
         .replace({"carrier": carrier_dict})
     )
 
+def Shapes2Shapes(orig, dest):
+    """
+    Adopted from vresutils.transfer.Shapes2Shapes()
+    """
+    orig_prepped = list(map(prep, orig))
+    transfer = sparse.lil_matrix((len(dest), len(orig)), dtype=float)
+
+    for i,j in product(range(len(dest)), range(len(orig))):
+        if orig_prepped[j].intersects(dest[i]):
+            area = orig[j].intersection(dest[i]).area
+            transfer[i,j] = area/dest[i].area
+
+    return transfer
+
 
 def attach_load(n, regions, load, nuts3_shapes, countries, scaling=1.0):
     substation_lv_i = n.buses.index[n.buses["substation_lv"]]
@@ -232,7 +248,7 @@ def attach_load(n, regions, load, nuts3_shapes, countries, scaling=1.0):
             return pd.DataFrame({group.index[0]: l})
         else:
             nuts3_cntry = nuts3.loc[nuts3.country == cntry]
-            transfer = vtransfer.Shapes2Shapes(
+            transfer = Shapes2Shapes(
                 group, nuts3_cntry.geometry, normed=False
             ).T.tocsr()
             gdp_n = pd.Series(

From e91068196b9db670ca95d00184e273f7890be61b Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Wed, 10 May 2023 10:02:41 +0200
Subject: [PATCH 081/176] replace vresutils annuity function with
 add_electricity.py calculate_annuity

---
 scripts/prepare_sector_network.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 1cb7146e..944c3caa 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -28,7 +28,7 @@ from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentati
 from pypsa.geo import haversine_pts
 from pypsa.io import import_components_from_dataframe
 from scipy.stats import beta
-from vresutils.costdata import annuity
+from add_electricity import calculate_annuity
 
 logger = logging.getLogger(__name__)
 
@@ -742,7 +742,7 @@ def prepare_costs(cost_file, config, nyears):
     costs = costs.fillna(config["fill_values"])
 
     def annuity_factor(v):
-        return annuity(v["lifetime"], v["discount rate"]) + v["FOM"] / 100
+        return calculate_annuity(v["lifetime"], v["discount rate"]) + v["FOM"] / 100
 
     costs["fixed"] = [
         annuity_factor(v) * v["investment"] * nyears for i, v in costs.iterrows()
@@ -851,7 +851,7 @@ def add_wave(n, wave_cost_factor):
     capacity = pd.Series({"Attenuator": 750, "F2HB": 1000, "MultiPA": 600})
 
     # in EUR/MW
-    annuity_factor = annuity(25, 0.07) + 0.03
+    annuity_factor = calculate_annuity(25, 0.07) + 0.03
     costs = (
         1e6
         * wave_cost_factor

From b21965a98600b4fcd3bbf286ac9b5f9bd9032fa6 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 10 May 2023 08:09:43 +0000
Subject: [PATCH 082/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/add_electricity.py        | 9 +++++----
 scripts/prepare_sector_network.py | 2 +-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 64cf442e..83ec4bb9 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -85,17 +85,17 @@ It further adds extendable ``generators`` with **zero** capacity for
 """
 
 import logging
+from itertools import product
 
 import geopandas as gpd
 import numpy as np
 import pandas as pd
 import powerplantmatching as pm
 import pypsa
+import scipy.sparse as sparse
 import xarray as xr
 from _helpers import configure_logging, update_p_nom_max
 from powerplantmatching.export import map_country_bus
-from itertools import product
-import scipy.sparse as sparse
 from shapely.prepared import prep
 
 idx = pd.IndexSlice
@@ -217,6 +217,7 @@ def load_powerplants(ppl_fn):
         .replace({"carrier": carrier_dict})
     )
 
+
 def Shapes2Shapes(orig, dest):
     """
     Adopted from vresutils.transfer.Shapes2Shapes()
@@ -224,10 +225,10 @@ def Shapes2Shapes(orig, dest):
     orig_prepped = list(map(prep, orig))
     transfer = sparse.lil_matrix((len(dest), len(orig)), dtype=float)
 
-    for i,j in product(range(len(dest)), range(len(orig))):
+    for i, j in product(range(len(dest)), range(len(orig))):
         if orig_prepped[j].intersects(dest[i]):
             area = orig[j].intersection(dest[i]).area
-            transfer[i,j] = area/dest[i].area
+            transfer[i, j] = area / dest[i].area
 
     return transfer
 
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 944c3caa..012c9714 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -22,13 +22,13 @@ from _helpers import (
     override_component_attrs,
     update_config_with_sector_opts,
 )
+from add_electricity import calculate_annuity
 from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2
 from networkx.algorithms import complement
 from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation
 from pypsa.geo import haversine_pts
 from pypsa.io import import_components_from_dataframe
 from scipy.stats import beta
-from add_electricity import calculate_annuity
 
 logger = logging.getLogger(__name__)
 

From 654f46f8683cf0613db90d38ab1127a9db62ffd4 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Thu, 11 May 2023 16:58:35 +0200
Subject: [PATCH 083/176] alpha testing complete vresutils removal

---
 envs/environment.fixed.yaml         |  3 +--
 envs/environment.yaml               |  3 +--
 scripts/add_electricity.py          |  6 ++----
 scripts/solve_network.py            | 31 ++++++++++++-----------------
 scripts/solve_operations_network.py | 30 ++++++++++++----------------
 5 files changed, 30 insertions(+), 43 deletions(-)

diff --git a/envs/environment.fixed.yaml b/envs/environment.fixed.yaml
index 7d8fcc45..1ff9313d 100644
--- a/envs/environment.fixed.yaml
+++ b/envs/environment.fixed.yaml
@@ -226,7 +226,7 @@ dependencies:
 - nspr=4.35
 - nss=3.88
 - numexpr=2.8.3
-- numpy=1.23.5
+- numpy=1.24
 - openjdk=17.0.3
 - openjpeg=2.5.0
 - openpyxl=3.1.0
@@ -378,4 +378,3 @@ dependencies:
   - highspy==1.5.0.dev0
   - pybind11==2.10.3
   - tsam==2.2.2
-  - vresutils==0.3.1
diff --git a/envs/environment.yaml b/envs/environment.yaml
index 0a9891a5..f970c9ba 100644
--- a/envs/environment.yaml
+++ b/envs/environment.yaml
@@ -25,7 +25,7 @@ dependencies:
 - pytables
 - lxml
 - powerplantmatching>=0.5.5
-- numpy<1.24
+- numpy
 - pandas>=1.4
 - geopandas>=0.11.0
 - xarray
@@ -55,5 +55,4 @@ dependencies:
 - rasterio!=1.2.10
 
 - pip:
-  - vresutils>=0.3.1
   - tsam>=1.1.0
diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 83ec4bb9..f910dee4 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -218,7 +218,7 @@ def load_powerplants(ppl_fn):
     )
 
 
-def Shapes2Shapes(orig, dest):
+def shapes_to_shapes(orig, dest):
     """
     Adopted from vresutils.transfer.Shapes2Shapes()
     """
@@ -249,9 +249,7 @@ def attach_load(n, regions, load, nuts3_shapes, countries, scaling=1.0):
             return pd.DataFrame({group.index[0]: l})
         else:
             nuts3_cntry = nuts3.loc[nuts3.country == cntry]
-            transfer = Shapes2Shapes(
-                group, nuts3_cntry.geometry, normed=False
-            ).T.tocsr()
+            transfer = shapes_to_shapes(group, nuts3_cntry.geometry).T.tocsr()
             gdp_n = pd.Series(
                 transfer.dot(nuts3_cntry["gdp"].fillna(1.0).values), index=group.index
             )
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index ff1c0ccf..53ac8816 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -38,7 +38,6 @@ from _helpers import (
     override_component_attrs,
     update_config_with_sector_opts,
 )
-from vresutils.benchmark import memory_logger
 
 logger = logging.getLogger(__name__)
 pypsa.pf.logger.setLevel(logging.WARNING)
@@ -667,23 +666,19 @@ if __name__ == "__main__":
 
     np.random.seed(solve_opts.get("seed", 123))
 
-    fn = getattr(snakemake.log, "memory", None)
-    with memory_logger(filename=fn, interval=30.0) as mem:
-        if "overrides" in snakemake.input.keys():
-            overrides = override_component_attrs(snakemake.input.overrides)
-            n = pypsa.Network(
-                snakemake.input.network, override_component_attrs=overrides
-            )
-        else:
-            n = pypsa.Network(snakemake.input.network)
-
-        n = prepare_network(n, solve_opts, config=snakemake.config)
-
-        n = solve_network(
-            n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver
+    if "overrides" in snakemake.input.keys():
+        overrides = override_component_attrs(snakemake.input.overrides)
+        n = pypsa.Network(
+            snakemake.input.network, override_component_attrs=overrides
         )
+    else:
+        n = pypsa.Network(snakemake.input.network)
 
-        n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
-        n.export_to_netcdf(snakemake.output[0])
+    n = prepare_network(n, solve_opts, config=snakemake.config)
 
-    logger.info("Maximum memory usage: {}".format(mem.mem_usage))
+    n = solve_network(
+        n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver
+    )
+
+    n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
+    n.export_to_netcdf(snakemake.output[0])
diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py
index 25fe0753..9c2fe3c1 100644
--- a/scripts/solve_operations_network.py
+++ b/scripts/solve_operations_network.py
@@ -46,23 +46,19 @@ if __name__ == "__main__":
 
     np.random.seed(solve_opts.get("seed", 123))
 
-    fn = getattr(snakemake.log, "memory", None)
-    with memory_logger(filename=fn, interval=30.0) as mem:
-        if "overrides" in snakemake.input:
-            overrides = override_component_attrs(snakemake.input.overrides)
-            n = pypsa.Network(
-                snakemake.input.network, override_component_attrs=overrides
-            )
-        else:
-            n = pypsa.Network(snakemake.input.network)
-
-        n.optimize.fix_optimal_capacities()
-        n = prepare_network(n, solve_opts, config=snakemake.config)
-        n = solve_network(
-            n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver
+    if "overrides" in snakemake.input:
+        overrides = override_component_attrs(snakemake.input.overrides)
+        n = pypsa.Network(
+            snakemake.input.network, override_component_attrs=overrides
         )
+    else:
+        n = pypsa.Network(snakemake.input.network)
 
-        n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
-        n.export_to_netcdf(snakemake.output[0])
+    n.optimize.fix_optimal_capacities()
+    n = prepare_network(n, solve_opts, config=snakemake.config)
+    n = solve_network(
+        n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver
+    )
 
-    logger.info("Maximum memory usage: {}".format(mem.mem_usage))
+    n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
+    n.export_to_netcdf(snakemake.output[0])

From 7d6d6d2805f34a6de6d6e1400d376e8970886c4d Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 11 May 2023 14:59:10 +0000
Subject: [PATCH 084/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/solve_network.py            | 4 +---
 scripts/solve_operations_network.py | 4 +---
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index 53ac8816..8f4fdf5b 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -668,9 +668,7 @@ if __name__ == "__main__":
 
     if "overrides" in snakemake.input.keys():
         overrides = override_component_attrs(snakemake.input.overrides)
-        n = pypsa.Network(
-            snakemake.input.network, override_component_attrs=overrides
-        )
+        n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
     else:
         n = pypsa.Network(snakemake.input.network)
 
diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py
index 9c2fe3c1..987302bb 100644
--- a/scripts/solve_operations_network.py
+++ b/scripts/solve_operations_network.py
@@ -48,9 +48,7 @@ if __name__ == "__main__":
 
     if "overrides" in snakemake.input:
         overrides = override_component_attrs(snakemake.input.overrides)
-        n = pypsa.Network(
-            snakemake.input.network, override_component_attrs=overrides
-        )
+        n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
     else:
         n = pypsa.Network(snakemake.input.network)
 

From 1758076815049e5e38b52859ba81132c21c446d2 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Thu, 11 May 2023 20:34:11 +0200
Subject: [PATCH 085/176] test param in add_electricity

---
 rules/build_electricity.smk |  8 ++++++
 scripts/add_electricity.py  | 50 ++++++++++++++++++-------------------
 2 files changed, 33 insertions(+), 25 deletions(-)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index 133ceb6e..194e186d 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -273,6 +273,14 @@ rule add_electricity:
         nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
     output:
         RESOURCES + "networks/elec.nc",
+    params:
+        costs=snakemake.config["costs"]
+        electricity=snakemake.config["electricity"]
+        renewable=snakemake.config["renewable"]
+        conventional=snakemake.config.get("conventional", {})
+        countries=snakemake.config["countries"]
+        scaling_factor=snakemake.config["load"]["scaling_factor"]
+        length_factor=snakemake.config["lines"]["length_factor"]
     log:
         LOGS + "add_electricity.log",
     benchmark:
diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 1d32bce1..bed5ee6b 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -686,15 +686,15 @@ def estimate_renewable_capacities(n, config):
             )
 
 
-def add_nice_carrier_names(n, config):
+def add_nice_carrier_names(n, plotting):
     carrier_i = n.carriers.index
     nice_names = (
-        pd.Series(config["plotting"]["nice_names"])
+        pd.Series(plotting["nice_names"])
         .reindex(carrier_i)
         .fillna(carrier_i.to_series().str.title())
     )
     n.carriers["nice_name"] = nice_names
-    colors = pd.Series(config["plotting"]["tech_colors"]).reindex(carrier_i)
+    colors = pd.Series(plotting["tech_colors"]).reindex(carrier_i)
     if colors.isna().any():
         missing_i = list(colors.index[colors.isna()])
         logger.warning(f"tech_colors for carriers {missing_i} not defined in config.")
@@ -713,23 +713,23 @@ if __name__ == "__main__":
 
     costs = load_costs(
         snakemake.input.tech_costs,
-        snakemake.config["costs"],
-        snakemake.config["electricity"],
+        snakemake.param["costs"],
+        snakemake.param["electricity"],
         Nyears,
     )
     ppl = load_powerplants(snakemake.input.powerplants)
 
-    if "renewable_carriers" in snakemake.config["electricity"]:
-        renewable_carriers = set(snakemake.config["electricity"]["renewable_carriers"])
+    if "renewable_carriers" in snakemake.param["electricity"]:
+        renewable_carriers = set(snakemake.param["electricity"]["renewable_carriers"])
     else:
         logger.warning(
             "Missing key `renewable_carriers` under config entry `electricity`. "
             "In future versions, this will raise an error. "
             "Falling back to carriers listed under `renewable`."
         )
-        renewable_carriers = snakemake.config["renewable"]
+        renewable_carriers = snakemake.param["renewable"]
 
-    extendable_carriers = snakemake.config["electricity"]["extendable_carriers"]
+    extendable_carriers = snakemake.param["electricity"]["extendable_carriers"]
     if not (set(renewable_carriers) & set(extendable_carriers["Generator"])):
         logger.warning(
             "No renewables found in config entry `extendable_carriers`. "
@@ -737,18 +737,18 @@ if __name__ == "__main__":
             "Falling back to all renewables."
         )
 
-    conventional_carriers = snakemake.config["electricity"]["conventional_carriers"]
+    conventional_carriers = snakemake.param["electricity"]["conventional_carriers"]
 
     attach_load(
         n,
         snakemake.input.regions,
         snakemake.input.load,
         snakemake.input.nuts3_shapes,
-        snakemake.config["countries"],
-        snakemake.config["load"]["scaling_factor"],
+        snakemake.param["countries"],
+        snakemake.param["load"]["scaling_factor"],
     )
 
-    update_transmission_costs(n, costs, snakemake.config["lines"]["length_factor"])
+    update_transmission_costs(n, costs, snakemake.param["lines"]["length_factor"])
 
     conventional_inputs = {
         k: v for k, v in snakemake.input.items() if k.startswith("conventional_")
@@ -759,7 +759,7 @@ if __name__ == "__main__":
         ppl,
         conventional_carriers,
         extendable_carriers,
-        snakemake.config.get("conventional", {}),
+        snakemake.param.get("conventional", {}),
         conventional_inputs,
     )
 
@@ -769,11 +769,11 @@ if __name__ == "__main__":
         snakemake.input,
         renewable_carriers,
         extendable_carriers,
-        snakemake.config["lines"]["length_factor"],
+        snakemake.param["lines"]["length_factor"],
     )
 
     if "hydro" in renewable_carriers:
-        conf = snakemake.config["renewable"]["hydro"]
+        conf = snakemake.param["renewable"]["hydro"]
         attach_hydro(
             n,
             costs,
@@ -784,7 +784,7 @@ if __name__ == "__main__":
             **conf,
         )
 
-    if "estimate_renewable_capacities" not in snakemake.config["electricity"]:
+    if "estimate_renewable_capacities" not in snakemake.param["electricity"]:
         logger.warning(
             "Missing key `estimate_renewable_capacities` under config entry `electricity`. "
             "In future versions, this will raise an error. "
@@ -792,18 +792,18 @@ if __name__ == "__main__":
         )
         if (
             "estimate_renewable_capacities_from_capacity_stats"
-            in snakemake.config["electricity"]
+            in snakemake.param["electricity"]
         ):
             estimate_renewable_caps = {
                 "enable": True,
-                **snakemake.config["electricity"][
+                **snakemake.param["electricity"][
                     "estimate_renewable_capacities_from_capacity_stats"
                 ],
             }
         else:
             estimate_renewable_caps = {"enable": False}
     else:
-        estimate_renewable_caps = snakemake.config["electricity"][
+        estimate_renewable_caps = snakemake.param["electricity"][
             "estimate_renewable_capacities"
         ]
     if "enable" not in estimate_renewable_caps:
@@ -819,21 +819,21 @@ if __name__ == "__main__":
             "Falling back to whether `renewable_capacities_from_opsd` is non-empty."
         )
         from_opsd = bool(
-            snakemake.config["electricity"].get("renewable_capacities_from_opsd", False)
+            snakemake.param["electricity"].get("renewable_capacities_from_opsd", False)
         )
         estimate_renewable_caps["from_opsd"] = from_opsd
 
     if estimate_renewable_caps["enable"]:
         if estimate_renewable_caps["from_opsd"]:
-            tech_map = snakemake.config["electricity"]["estimate_renewable_capacities"][
+            tech_map = snakemake.param["electricity"]["estimate_renewable_capacities"][
                 "technology_mapping"
             ]
             attach_OPSD_renewables(n, tech_map)
-        estimate_renewable_capacities(n, snakemake.config)
+        estimate_renewable_capacities(n, snakemake.param)
 
     update_p_nom_max(n)
 
-    add_nice_carrier_names(n, snakemake.config)
+    add_nice_carrier_names(n, snakemake.param["plotting"])
 
-    n.meta = snakemake.config
+    n.meta = snakemake.param
     n.export_to_netcdf(snakemake.output[0])

From aa50ea44cc550433b8db38bb241c359b8ccc92ce Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Thu, 11 May 2023 20:59:37 +0200
Subject: [PATCH 086/176] fix add_electricity mistake

---
 rules/build_electricity.smk | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index 194e186d..8b7780db 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -274,13 +274,13 @@ rule add_electricity:
     output:
         RESOURCES + "networks/elec.nc",
     params:
-        costs=snakemake.config["costs"]
-        electricity=snakemake.config["electricity"]
-        renewable=snakemake.config["renewable"]
-        conventional=snakemake.config.get("conventional", {})
-        countries=snakemake.config["countries"]
-        scaling_factor=snakemake.config["load"]["scaling_factor"]
-        length_factor=snakemake.config["lines"]["length_factor"]
+        costs=snakemake.config["costs"],
+        electricity=snakemake.config["electricity"],
+        renewable=snakemake.config["renewable"],
+        conventional=snakemake.config.get("conventional", {}),
+        countries=snakemake.config["countries"],
+        scaling_factor=snakemake.config["load"]["scaling_factor"],
+        length_factor=snakemake.config["lines"]["length_factor"],
     log:
         LOGS + "add_electricity.log",
     benchmark:

From e29571535b504c2db3733e877ca0d0c70248b07f Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Fri, 12 May 2023 07:11:41 +0200
Subject: [PATCH 087/176] remove references to memory logfile in *.smk, add
 release note

---
 doc/release_notes.rst               | 2 ++
 doc/requirements.txt                | 1 -
 rules/solve_electricity.smk         | 4 ----
 rules/solve_myopic.smk              | 2 --
 rules/solve_overnight.smk           | 2 --
 scripts/solve_operations_network.py | 1 -
 6 files changed, 2 insertions(+), 10 deletions(-)

diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index 8e6bfdd4..14f2939d 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -23,6 +23,8 @@ Upcoming Release
   hydrogen fuel cell. Add switches for both re-electrification options under
   ``sector: hydrogen_turbine:`` and ``sector: hydrogen_fuel_cell:``.
 
+* Remove ``vresutils`` dependency.
+
 PyPSA-Eur 0.8.0 (18th March 2023)
 =================================
 
diff --git a/doc/requirements.txt b/doc/requirements.txt
index 2f08b8d9..3e760c81 100644
--- a/doc/requirements.txt
+++ b/doc/requirements.txt
@@ -9,7 +9,6 @@ sphinxcontrib-bibtex
 myst-parser  # recommark is deprecated, https://stackoverflow.com/a/71660856/13573820
 
 pypsa
-vresutils>=0.3.1
 powerplantmatching>=0.5.5
 atlite>=0.2.9
 dask[distributed]
diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk
index 8ddeca92..fc70ce42 100644
--- a/rules/solve_electricity.smk
+++ b/rules/solve_electricity.smk
@@ -14,8 +14,6 @@ rule solve_network:
         ),
         python=LOGS
         + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log",
-        memory=LOGS
-        + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_memory.log",
     benchmark:
         BENCHMARKS + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}"
     threads: 4
@@ -41,8 +39,6 @@ rule solve_operations_network:
         ),
         python=LOGS
         + "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log",
-        memory=LOGS
-        + "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_memory.log",
     benchmark:
         (
             BENCHMARKS
diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk
index f10d8157..ec4281ff 100644
--- a/rules/solve_myopic.smk
+++ b/rules/solve_myopic.smk
@@ -90,8 +90,6 @@ rule solve_sector_network_myopic:
         + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log",
         python=LOGS
         + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log",
-        memory=LOGS
-        + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_memory.log",
     threads: 4
     resources:
         mem_mb=config["solving"]["mem"],
diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk
index c2e103e5..f05925b0 100644
--- a/rules/solve_overnight.smk
+++ b/rules/solve_overnight.smk
@@ -21,8 +21,6 @@ rule solve_sector_network:
         + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log",
         python=LOGS
         + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log",
-        memory=LOGS
-        + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_memory.log",
     threads: config["solving"]["solver"].get("threads", 4)
     resources:
         mem_mb=config["solving"]["mem"],
diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py
index 987302bb..27520485 100644
--- a/scripts/solve_operations_network.py
+++ b/scripts/solve_operations_network.py
@@ -17,7 +17,6 @@ from _helpers import (
     update_config_with_sector_opts,
 )
 from solve_network import prepare_network, solve_network
-from vresutils.benchmark import memory_logger
 
 logger = logging.getLogger(__name__)
 

From 5be21dfc5a4cefd5e01bbf7798663d18eeacc638 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Fri, 12 May 2023 09:07:16 +0200
Subject: [PATCH 088/176] add option for piecewise linear transmission loss
 approximation

---
 config/config.default.yaml   | 2 +-
 doc/configtables/solving.csv | 2 +-
 doc/release_notes.rst        | 4 ++++
 envs/environment.yaml        | 2 +-
 scripts/solve_network.py     | 3 +++
 5 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/config/config.default.yaml b/config/config.default.yaml
index 10e6a6ed..78c1385c 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -623,9 +623,9 @@ clustering:
 solving:
   #tmpdir: "path/to/tmp"
   options:
-    formulation: kirchhoff
     clip_p_max_pu: 1.e-2
     load_shedding: false
+    transmission_losses: 0
     noisy_costs: true
     skip_iterations: true
     track_iterations: false
diff --git a/doc/configtables/solving.csv b/doc/configtables/solving.csv
index cba28cbe..c252ff32 100644
--- a/doc/configtables/solving.csv
+++ b/doc/configtables/solving.csv
@@ -1,7 +1,7 @@
 ,Unit,Values,Description
 options,,,
--- formulation,--,"Any of {'angles', 'kirchhoff', 'cycles', 'ptdf'}","Specifies which variant of linearized power flow formulations to use in the optimisation problem. Recommended is 'kirchhoff'. Explained in `this article <https://arxiv.org/abs/1704.01881>`_."
 -- load_shedding,bool/float,"{'true','false', float}","Add generators with very high marginal cost to simulate load shedding and avoid problem infeasibilities. If load shedding is a float, it denotes the marginal cost in EUR/kWh."
+-- transmission_losses,int,"[0-9]","Add piecewise linear approximation of transmission losses based on n tangents. Defaults to 0, which means losses are ignored."
 -- noisy_costs,bool,"{'true','false'}","Add random noise to marginal cost of generators by :math:`\mathcal{U}(0.009,0,011)` and capital cost of lines and links by :math:`\mathcal{U}(0.09,0,11)`."
 -- min_iterations,--,int,"Minimum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run."
 -- max_iterations,--,int,"Maximum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run."
diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index 14f2939d..3af16477 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -25,6 +25,10 @@ Upcoming Release
 
 * Remove ``vresutils`` dependency.
 
+* Add option to include a piecewise linear approximation of transmission losses,
+  e.g. by setting ``solving: options: transmission_losses: 2`` for an
+  approximation with two tangents.
+
 PyPSA-Eur 0.8.0 (18th March 2023)
 =================================
 
diff --git a/envs/environment.yaml b/envs/environment.yaml
index f970c9ba..9d800fdc 100644
--- a/envs/environment.yaml
+++ b/envs/environment.yaml
@@ -10,7 +10,7 @@ dependencies:
 - python>=3.8
 - pip
 
-- pypsa>=0.21.3
+- pypsa>=0.23
 - atlite>=0.2.9
 - dask
 
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index 8f4fdf5b..e671ffd3 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -600,6 +600,7 @@ def solve_network(n, config, opts="", **kwargs):
     track_iterations = cf_solving.get("track_iterations", False)
     min_iterations = cf_solving.get("min_iterations", 4)
     max_iterations = cf_solving.get("max_iterations", 6)
+    transmission_losses = cf_solving.get("transmission_losses", 0)
 
     # add to network for extra_functionality
     n.config = config
@@ -613,6 +614,7 @@ def solve_network(n, config, opts="", **kwargs):
     if skip_iterations:
         status, condition = n.optimize(
             solver_name=solver_name,
+            transmission_losses=transmission_losses,
             extra_functionality=extra_functionality,
             **solver_options,
             **kwargs,
@@ -623,6 +625,7 @@ def solve_network(n, config, opts="", **kwargs):
             track_iterations=track_iterations,
             min_iterations=min_iterations,
             max_iterations=max_iterations,
+            transmission_losses=transmission_losses,
             extra_functionality=extra_functionality,
             **solver_options,
             **kwargs,

From 99963edaaf7c6b7b6d70fe586a15eb4770d3e166 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Fri, 12 May 2023 09:29:38 +0200
Subject: [PATCH 089/176] add option to limit maximum extension per line/link;
 linemaxext

---
 config/config.default.yaml        |  2 ++
 doc/configtables/lines.csv        |  1 +
 doc/configtables/links.csv        |  1 +
 doc/release_notes.rst             |  4 ++++
 scripts/prepare_network.py        | 18 +++++++++++++++---
 scripts/prepare_sector_network.py |  1 -
 6 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/config/config.default.yaml b/config/config.default.yaml
index 10e6a6ed..5733b942 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -264,12 +264,14 @@ lines:
     380.: "Al/St 240/40 4-bundle 380.0"
   s_max_pu: 0.7
   s_nom_max: .inf
+  max_extension: .inf
   length_factor: 1.25
   under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity
 
 links:
   p_max_pu: 1.0
   p_nom_max: .inf
+  max_extension: .inf
   include_tyndp: true
   under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity
 
diff --git a/doc/configtables/lines.csv b/doc/configtables/lines.csv
index ddf02e54..ad5dd690 100644
--- a/doc/configtables/lines.csv
+++ b/doc/configtables/lines.csv
@@ -2,5 +2,6 @@
 types,--,"Values should specify a `line type in PyPSA <https://pypsa.readthedocs.io/en/latest/components.html#line-types>`_. Keys should specify the corresponding voltage level (e.g. 220., 300. and 380. kV)","Specifies line types to assume for the different voltage levels of the ENTSO-E grid extraction. Should normally handle voltage levels 220, 300, and 380 kV"
 s_max_pu,--,"Value in [0.,1.]","Correction factor for line capacities (``s_nom``) to approximate :math:`N-1` security and reserve capacity for reactive power flows"
 s_nom_max,MW,"float","Global upper limit for the maximum capacity of each extendable line."
+max_extension,MW,"float","Upper limit for the extended capacity of each extendable line."
 length_factor,--,float,"Correction factor to account for the fact that buses are *not* connected by lines through air-line distance."
 under_construction,--,"One of {'zero': set capacity to zero, 'remove': remove completely, 'keep': keep with full capacity}","Specifies how to handle lines which are currently under construction."
diff --git a/doc/configtables/links.csv b/doc/configtables/links.csv
index c9981dc7..c1ffb427 100644
--- a/doc/configtables/links.csv
+++ b/doc/configtables/links.csv
@@ -1,5 +1,6 @@
 ,Unit,Values,Description
 p_max_pu,--,"Value in [0.,1.]","Correction factor for link capacities ``p_nom``."
 p_nom_max,MW,"float","Global upper limit for the maximum capacity of each extendable DC link."
+max_extension,MW,"float","Upper limit for the extended capacity of each extendable DC link."
 include_tyndp,bool,"{'true', 'false'}","Specifies whether to add HVDC link projects from the `TYNDP 2018 <https://tyndp.entsoe.eu/tyndp2018/projects/>`_ which are at least in permitting."
 under_construction,--,"One of {'zero': set capacity to zero, 'remove': remove completely, 'keep': keep with full capacity}","Specifies how to handle lines which are currently under construction."
diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index 14f2939d..818227d8 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -25,6 +25,10 @@ Upcoming Release
 
 * Remove ``vresutils`` dependency.
 
+* Added configuration option ``lines: max_extension:`` and ``links:
+  max_extension:``` to control the maximum capacity addition per line or link in
+  MW.
+
 PyPSA-Eur 0.8.0 (18th March 2023)
 =================================
 
diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index 14a003ae..01b72605 100755
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -233,7 +233,17 @@ def enforce_autarky(n, only_crossborder=False):
     n.mremove("Link", links_rm)
 
 
-def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf):
+def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf, s_nom_max_ext=np.inf, p_nom_max_ext=np.inf):
+
+    if np.isfinite(s_nom_max_ext) and s_nom_max_ext > 0:
+        logger.info(f"Limiting line extensions to {s_nom_max_ext} MW")
+        n.lines["s_nom_max"] = n.lines["s_nom"] + s_nom_max_ext
+
+    if np.isfinite(p_nom_max_ext) and p_nom_max_ext > 0:
+        logger.info(f"Limiting line extensions to {p_nom_max_ext} MW")
+        hvdc = n.links.index[n.links.carrier == "DC"]
+        n.links.loc[hvdc, "p_nom_max"] = n.links.loc[hvdc, "p_nom"] + p_nom_max_ext
+
     n.lines.s_nom_max.clip(upper=s_nom_max_set, inplace=True)
     n.links.p_nom_max.clip(upper=p_nom_max_set, inplace=True)
 
@@ -330,8 +340,10 @@ if __name__ == "__main__":
 
     set_line_nom_max(
         n,
-        s_nom_max_set=snakemake.config["lines"].get("s_nom_max,", np.inf),
-        p_nom_max_set=snakemake.config["links"].get("p_nom_max,", np.inf),
+        s_nom_max_set=snakemake.config["lines"]["s_nom_max,"],
+        p_nom_max_set=snakemake.config["links"]["p_nom_max,"],
+        s_nom_max_ext=snakemake.config["lines"]["max_extension,"],
+        p_nom_max_ext=snakemake.config["links"]["max_extension,"],
     )
 
     if "ATK" in opts:
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 012c9714..3009bfd9 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -3052,7 +3052,6 @@ def maybe_adjust_costs_and_potentials(n, opts):
             logger.info(f"changing {attr} for {carrier} by factor {factor}")
 
 
-# TODO this should rather be a config no wildcard
 def limit_individual_line_extension(n, maxext):
     logger.info(f"Limiting new HVAC and HVDC extensions to {maxext} MW")
     n.lines["s_nom_max"] = n.lines["s_nom"] + maxext

From 3343394c65e4268b0c375fd3024daff9d6f94c60 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 12 May 2023 07:30:10 +0000
Subject: [PATCH 090/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/prepare_network.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index 01b72605..e4bc9705 100755
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -233,8 +233,13 @@ def enforce_autarky(n, only_crossborder=False):
     n.mremove("Link", links_rm)
 
 
-def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf, s_nom_max_ext=np.inf, p_nom_max_ext=np.inf):
-
+def set_line_nom_max(
+    n,
+    s_nom_max_set=np.inf,
+    p_nom_max_set=np.inf,
+    s_nom_max_ext=np.inf,
+    p_nom_max_ext=np.inf,
+):
     if np.isfinite(s_nom_max_ext) and s_nom_max_ext > 0:
         logger.info(f"Limiting line extensions to {s_nom_max_ext} MW")
         n.lines["s_nom_max"] = n.lines["s_nom"] + s_nom_max_ext

From e88b57f42250f062768bc12e6b4ae0889f8414e2 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Fri, 12 May 2023 10:42:06 +0200
Subject: [PATCH 091/176] prepare_network: fix typo

---
 scripts/prepare_network.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index e4bc9705..e16339ba 100755
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -345,10 +345,10 @@ if __name__ == "__main__":
 
     set_line_nom_max(
         n,
-        s_nom_max_set=snakemake.config["lines"]["s_nom_max,"],
-        p_nom_max_set=snakemake.config["links"]["p_nom_max,"],
-        s_nom_max_ext=snakemake.config["lines"]["max_extension,"],
-        p_nom_max_ext=snakemake.config["links"]["max_extension,"],
+        s_nom_max_set=snakemake.config["lines"]["s_nom_max"],
+        p_nom_max_set=snakemake.config["links"]["p_nom_max"],
+        s_nom_max_ext=snakemake.config["lines"]["max_extension"],
+        p_nom_max_ext=snakemake.config["links"]["max_extension"],
     )
 
     if "ATK" in opts:

From 67ef38b35eb2e46d8dcc5252216b73ee062465f0 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Fri, 12 May 2023 13:59:41 +0200
Subject: [PATCH 092/176] retrieve*: handle extraction path via
 snakemake.output

---
 scripts/retrieve_databundle.py              |  3 +--
 scripts/retrieve_gas_infrastructure_data.py |  2 +-
 scripts/retrieve_sector_databundle.py       | 16 +++++++++-------
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py
index 0c6a7feb..75d8519e 100644
--- a/scripts/retrieve_databundle.py
+++ b/scripts/retrieve_databundle.py
@@ -58,9 +58,8 @@ if __name__ == "__main__":
     else:
         url = "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz"
 
-    # Save locations
     tarball_fn = Path(f"{rootpath}/bundle.tar.xz")
-    to_fn = Path(f"{rootpath}/data")
+    to_fn = Path(rootpath) / Path(snakemake.output[0]).parent.parent
 
     logger.info(f"Downloading databundle from '{url}'.")
     disable_progress = snakemake.config["run"].get("disable_progressbar", False)
diff --git a/scripts/retrieve_gas_infrastructure_data.py b/scripts/retrieve_gas_infrastructure_data.py
index dda7bd8c..42b726db 100644
--- a/scripts/retrieve_gas_infrastructure_data.py
+++ b/scripts/retrieve_gas_infrastructure_data.py
@@ -29,7 +29,7 @@ if __name__ == "__main__":
 
     # Save locations
     zip_fn = Path(f"{rootpath}/IGGIELGN.zip")
-    to_fn = Path(f"{rootpath}/data/gas_network/scigrid-gas")
+    to_fn = Path(rootpath) / Path(snakemake.output[0]).parent.parent
 
     logger.info(f"Downloading databundle from '{url}'.")
     disable_progress = snakemake.config["run"].get("disable_progressbar", False)
diff --git a/scripts/retrieve_sector_databundle.py b/scripts/retrieve_sector_databundle.py
index 97426ab2..0991bbe3 100644
--- a/scripts/retrieve_sector_databundle.py
+++ b/scripts/retrieve_sector_databundle.py
@@ -10,23 +10,25 @@ import logging
 
 logger = logging.getLogger(__name__)
 
-import os
-import sys
 import tarfile
 from pathlib import Path
 
-# Add pypsa-eur scripts to path for import of _helpers
-sys.path.insert(0, os.getcwd() + "/../pypsa-eur/scripts")
-
 from _helpers import configure_logging, progress_retrieve
 
 if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake("retrieve_databundle")
+        rootpath = ".."
+    else:
+        rootpath = "."
     configure_logging(snakemake)
 
     url = "https://zenodo.org/record/5824485/files/pypsa-eur-sec-data-bundle.tar.gz"
 
-    tarball_fn = Path("sector-bundle.tar.gz")
-    to_fn = Path("data")
+    tarball_fn = Path(f"{rootpath}/sector-bundle.tar.gz")
+    to_fn = Path(rootpath) / Path(snakemake.output[0]).parent
 
     logger.info(f"Downloading databundle from '{url}'.")
     disable_progress = snakemake.config["run"].get("disable_progressbar", False)

From b2216355f10bf11147cc7f03e74b17f6dc4b60e6 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Fri, 12 May 2023 14:10:04 +0200
Subject: [PATCH 093/176] retrieve_sector_data: add another .parent

---
 scripts/retrieve_sector_databundle.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/retrieve_sector_databundle.py b/scripts/retrieve_sector_databundle.py
index 0991bbe3..0d172c8d 100644
--- a/scripts/retrieve_sector_databundle.py
+++ b/scripts/retrieve_sector_databundle.py
@@ -28,7 +28,7 @@ if __name__ == "__main__":
     url = "https://zenodo.org/record/5824485/files/pypsa-eur-sec-data-bundle.tar.gz"
 
     tarball_fn = Path(f"{rootpath}/sector-bundle.tar.gz")
-    to_fn = Path(rootpath) / Path(snakemake.output[0]).parent
+    to_fn = Path(rootpath) / Path(snakemake.output[0]).parent.parent
 
     logger.info(f"Downloading databundle from '{url}'.")
     disable_progress = snakemake.config["run"].get("disable_progressbar", False)

From 8f91963e7623d6cf574c0fd267b80b581f9560d8 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Fri, 12 May 2023 14:50:58 +0200
Subject: [PATCH 094/176] properly reference p_max_pu files

---
 scripts/add_electricity.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index f910dee4..85ab35e2 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -418,7 +418,7 @@ def attach_conventional_generators(
             if f"conventional_{carrier}_{attr}" in conventional_inputs:
                 # Values affecting generators of technology k country-specific
                 # First map generator buses to countries; then map countries to p_max_pu
-                values = pd.read_csv(values, index_col=0).iloc[:, 0]
+                values = pd.read_csv(snakemake.input[f"conventional_{carrier}_{attr}"], index_col=0).iloc[:, 0]
                 bus_values = n.buses.country.map(values)
                 n.generators[attr].update(
                     n.generators.loc[idx].bus.map(bus_values).dropna()

From 38bae672da42db241eb885eceeb3a6097644eeb5 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 12 May 2023 12:51:39 +0000
Subject: [PATCH 095/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/add_electricity.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 85ab35e2..4afa3e22 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -418,7 +418,9 @@ def attach_conventional_generators(
             if f"conventional_{carrier}_{attr}" in conventional_inputs:
                 # Values affecting generators of technology k country-specific
                 # First map generator buses to countries; then map countries to p_max_pu
-                values = pd.read_csv(snakemake.input[f"conventional_{carrier}_{attr}"], index_col=0).iloc[:, 0]
+                values = pd.read_csv(
+                    snakemake.input[f"conventional_{carrier}_{attr}"], index_col=0
+                ).iloc[:, 0]
                 bus_values = n.buses.country.map(values)
                 n.generators[attr].update(
                     n.generators.loc[idx].bus.map(bus_values).dropna()

From 1fc48d8753f93b2b2d2f24c2516cf16777d72cb4 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Mon, 15 May 2023 10:33:17 +0200
Subject: [PATCH 096/176] using python algorithm to edit rules and script

---
 rules/build_electricity.smk                   | 55 +++++++++++++---
 rules/build_sector.smk                        | 65 +++++++++++++++++++
 rules/postprocess.smk                         | 13 ++++
 rules/retrieve.smk                            |  2 +
 rules/solve_electricity.smk                   |  4 ++
 rules/solve_myopic.smk                        | 12 ++++
 rules/solve_overnight.smk                     |  2 +
 scripts/_helpers.py                           |  2 +-
 scripts/add_brownfield.py                     |  8 +--
 scripts/add_electricity.py                    | 50 +++++++-------
 scripts/add_existing_baseyear.py              | 20 +++---
 scripts/add_extra_components.py               |  4 +-
 scripts/build_ammonia_production.py           |  2 +-
 scripts/build_biomass_potentials.py           |  2 +-
 scripts/build_bus_regions.py                  |  2 +-
 scripts/build_cop_profiles.py                 |  2 +-
 scripts/build_cutout.py                       |  4 +-
 scripts/build_electricity_demand.py           | 12 ++--
 scripts/build_energy_totals.py                | 12 ++--
 scripts/build_gas_input_locations.py          |  2 +-
 scripts/build_gas_network.py                  |  2 +-
 scripts/build_heat_demand.py                  |  2 +-
 scripts/build_hydro_profile.py                |  4 +-
 scripts/build_industrial_distribution_key.py  |  6 +-
 ...ustrial_energy_demand_per_country_today.py |  6 +-
 ...build_industrial_production_per_country.py | 10 +--
 ...ustrial_production_per_country_tomorrow.py |  2 +-
 scripts/build_industry_sector_ratios.py       |  4 +-
 scripts/build_population_layouts.py           |  2 +-
 scripts/build_powerplants.py                  |  6 +-
 scripts/build_renewable_profiles.py           |  4 +-
 scripts/build_retro_cost.py                   |  4 +-
 scripts/build_sequestration_potentials.py     |  2 +-
 scripts/build_shapes.py                       |  4 +-
 scripts/build_solar_thermal_profiles.py       |  4 +-
 scripts/build_temperature_profiles.py         |  2 +-
 scripts/build_transport_demand.py             |  4 +-
 scripts/cluster_gas_network.py                |  2 +-
 scripts/cluster_network.py                    | 18 ++---
 scripts/make_summary.py                       | 22 +++----
 scripts/plot_network.py                       | 16 ++---
 scripts/plot_summary.py                       | 32 ++++-----
 scripts/prepare_network.py                    | 20 +++---
 scripts/prepare_sector_network.py             | 42 ++++++------
 scripts/retrieve_databundle.py                |  4 +-
 scripts/simplify_network.py                   |  8 +--
 scripts/solve_network.py                      |  2 +-
 scripts/solve_operations_network.py           |  2 +-
 48 files changed, 324 insertions(+), 187 deletions(-)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index 8b7780db..bb000f2b 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -19,6 +19,10 @@ if config["enable"].get("prepare_links_p_nom", False):
 
 
 rule build_electricity_demand:
+    params:
+        snapshots=config["snapshots"],
+        countries=config["countries"],
+        load=config["load"],
     input:
         ancient("data/load_raw.csv"),
     output:
@@ -34,6 +38,9 @@ rule build_electricity_demand:
 
 
 rule build_powerplants:
+    params:
+        electricity=config["electricity"],
+        countries=config["countries"],
     input:
         base_network=RESOURCES + "networks/base.nc",
         custom_powerplants="data/custom_powerplants.csv",
@@ -79,6 +86,8 @@ rule base_network:
 
 
 rule build_shapes:
+    params:
+        countries=config["countries"],
     input:
         naturalearth=ancient("data/bundle/naturalearth/ne_10m_admin_0_countries.shp"),
         eez=ancient("data/bundle/eez/World_EEZ_v8_2014.shp"),
@@ -104,6 +113,8 @@ rule build_shapes:
 
 
 rule build_bus_regions:
+    params:
+        countries=config["countries"],
     input:
         country_shapes=RESOURCES + "country_shapes.geojson",
         offshore_shapes=RESOURCES + "offshore_shapes.geojson",
@@ -125,6 +136,9 @@ rule build_bus_regions:
 if config["enable"].get("build_cutout", False):
 
     rule build_cutout:
+        params:
+            snapshots=config["snapshots"],
+            atlite=config["atlite"],
         input:
             regions_onshore=RESOURCES + "regions_onshore.geojson",
             regions_offshore=RESOURCES + "regions_offshore.geojson",
@@ -186,6 +200,8 @@ rule build_ship_raster:
 
 
 rule build_renewable_profiles:
+    params:
+        renewable=config["renewable"],
     input:
         base_network=RESOURCES + "networks/base.nc",
         corine=ancient("data/bundle/corine/g250_clc06_V18_5.tif"),
@@ -235,6 +251,9 @@ rule build_renewable_profiles:
 
 
 rule build_hydro_profile:
+    params:
+        countries=config["countries"],
+        renewable=config["renewable"],
     input:
         country_shapes=RESOURCES + "country_shapes.geojson",
         eia_hydro_generation="data/eia_hydro_annual_generation.csv",
@@ -252,6 +271,13 @@ rule build_hydro_profile:
 
 
 rule add_electricity:
+    params:
+        lines=config["lines"],
+        load=config["load"],
+        countries=config["countries"],
+        renewable=config["renewable"],
+        electricity=config["electricity"],
+        costs=config["costs"],
     input:
         **{
             f"profile_{tech}": RESOURCES + f"profile_{tech}.nc"
@@ -273,14 +299,6 @@ rule add_electricity:
         nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
     output:
         RESOURCES + "networks/elec.nc",
-    params:
-        costs=snakemake.config["costs"],
-        electricity=snakemake.config["electricity"],
-        renewable=snakemake.config["renewable"],
-        conventional=snakemake.config.get("conventional", {}),
-        countries=snakemake.config["countries"],
-        scaling_factor=snakemake.config["load"]["scaling_factor"],
-        length_factor=snakemake.config["lines"]["length_factor"],
     log:
         LOGS + "add_electricity.log",
     benchmark:
@@ -295,6 +313,10 @@ rule add_electricity:
 
 
 rule simplify_network:
+    params:
+        clustering=config["clustering"],
+        electricity=config["electricity"],
+        costs=config["costs"],
     input:
         network=RESOURCES + "networks/elec.nc",
         tech_costs=COSTS,
@@ -320,6 +342,14 @@ rule simplify_network:
 
 
 rule cluster_network:
+    params:
+        solving=config["solving"],
+        electricity=config["electricity"],
+        costs=config["costs"],
+        lines=config["lines"],
+        renewable=config["renewable"],
+        clustering=config["clustering"],
+        enable=config["enable"],
     input:
         network=RESOURCES + "networks/elec_s{simpl}.nc",
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson",
@@ -351,6 +381,9 @@ rule cluster_network:
 
 
 rule add_extra_components:
+    params:
+        costs=config["costs"],
+        electricity=config["electricity"],
     input:
         network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc",
         tech_costs=COSTS,
@@ -370,6 +403,12 @@ rule add_extra_components:
 
 
 rule prepare_network:
+    params:
+        links=config["links"],
+        solving=config["solving"],
+        lines=config["lines"],
+        electricity=config["electricity"],
+        costs=config["costs"],
     input:
         RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc",
         tech_costs=COSTS,
diff --git a/rules/build_sector.smk b/rules/build_sector.smk
index 1b724d1a..d375b7b9 100644
--- a/rules/build_sector.smk
+++ b/rules/build_sector.smk
@@ -4,6 +4,8 @@
 
 
 rule build_population_layouts:
+    params:
+        logging=config["logging"],
     input:
         nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
         urban_percent="data/urban_percent.csv",
@@ -70,6 +72,8 @@ rule build_simplified_population_layouts:
 if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
 
     rule build_gas_network:
+        params:
+            logging=config["logging"],
         input:
             gas_network="data/gas_network/scigrid-gas/data/IGGIELGN_PipeSegments.geojson",
         output:
@@ -84,6 +88,8 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
             "../scripts/build_gas_network.py"
 
     rule build_gas_input_locations:
+        params:
+            logging=config["logging"],
         input:
             lng=HTTP.remote(
                 "https://globalenergymonitor.org/wp-content/uploads/2022/09/Europe-Gas-Tracker-August-2022.xlsx",
@@ -110,6 +116,8 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
             "../scripts/build_gas_input_locations.py"
 
     rule cluster_gas_network:
+        params:
+            logging=config["logging"],
         input:
             cleaned_gas_network=RESOURCES + "gas_network.csv",
             regions_onshore=RESOURCES
@@ -140,6 +148,8 @@ if not (config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]):
 
 
 rule build_heat_demands:
+    params:
+        snapshots=config["snapshots"],
     input:
         pop_layout=RESOURCES + "pop_layout_{scope}.nc",
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
@@ -160,6 +170,8 @@ rule build_heat_demands:
 
 
 rule build_temperature_profiles:
+    params:
+        snapshots=config["snapshots"],
     input:
         pop_layout=RESOURCES + "pop_layout_{scope}.nc",
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
@@ -181,6 +193,8 @@ rule build_temperature_profiles:
 
 
 rule build_cop_profiles:
+    params:
+        sector=config["sector"],
     input:
         temp_soil_total=RESOURCES + "temp_soil_total_elec_s{simpl}_{clusters}.nc",
         temp_soil_rural=RESOURCES + "temp_soil_rural_elec_s{simpl}_{clusters}.nc",
@@ -208,6 +222,9 @@ rule build_cop_profiles:
 
 
 rule build_solar_thermal_profiles:
+    params:
+        snapshots=config["snapshots"],
+        solar_thermal=config["solar_thermal"],
     input:
         pop_layout=RESOURCES + "pop_layout_{scope}.nc",
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
@@ -228,6 +245,11 @@ rule build_solar_thermal_profiles:
 
 
 rule build_energy_totals:
+    params:
+        run=config["run"],
+        countries=config["countries"],
+        energy=config["energy"],
+        logging=config["logging"],
     input:
         nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
         co2="data/eea/UNFCCC_v23.csv",
@@ -253,6 +275,8 @@ rule build_energy_totals:
 
 
 rule build_biomass_potentials:
+    params:
+        biomass=config["biomass"],
     input:
         enspreso_biomass=HTTP.remote(
             "https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx",
@@ -315,6 +339,8 @@ if not config["sector"]["biomass_transport"]:
 if config["sector"]["regional_co2_sequestration_potential"]["enable"]:
 
     rule build_sequestration_potentials:
+        params:
+            sector=config["sector"],
         input:
             sequestration_potential=HTTP.remote(
                 "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson",
@@ -368,6 +394,8 @@ rule build_salt_cavern_potentials:
 
 
 rule build_ammonia_production:
+    params:
+        countries=config["countries"],
     input:
         usgs="data/myb1-2017-nitro.xls",
     output:
@@ -386,6 +414,9 @@ rule build_ammonia_production:
 
 
 rule build_industry_sector_ratios:
+    params:
+        industry=config["industry"],
+        sector=config["sector"],
     input:
         ammonia_production=RESOURCES + "ammonia_production.csv",
         idees="data/jrc-idees-2015",
@@ -405,6 +436,11 @@ rule build_industry_sector_ratios:
 
 
 rule build_industrial_production_per_country:
+    params:
+        run=config["run"],
+        industry=config["industry"],
+        countries=config["countries"],
+        logging=config["logging"],
     input:
         ammonia_production=RESOURCES + "ammonia_production.csv",
         jrc="data/jrc-idees-2015",
@@ -426,6 +462,8 @@ rule build_industrial_production_per_country:
 
 
 rule build_industrial_production_per_country_tomorrow:
+    params:
+        industry=config["industry"],
     input:
         industrial_production_per_country=RESOURCES
         + "industrial_production_per_country.csv",
@@ -450,6 +488,10 @@ rule build_industrial_production_per_country_tomorrow:
 
 
 rule build_industrial_distribution_key:
+    params:
+        industry=config["industry"],
+        countries=config["countries"],
+        logging=config["logging"],
     input:
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
         clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv",
@@ -524,6 +566,10 @@ rule build_industrial_energy_demand_per_node:
 
 
 rule build_industrial_energy_demand_per_country_today:
+    params:
+        run=config["run"],
+        countries=config["countries"],
+        industry=config["industry"],
     input:
         jrc="data/jrc-idees-2015",
         ammonia_production=RESOURCES + "ammonia_production.csv",
@@ -570,6 +616,9 @@ rule build_industrial_energy_demand_per_node_today:
 if config["sector"]["retrofitting"]["retro_endogen"]:
 
     rule build_retro_cost:
+        params:
+            sector=config["sector"],
+            countries=config["countries"],
         input:
             building_stock="data/retro/data_building_stock.csv",
             data_tabula="data/retro/tabula-calculator-calcsetbuilding.csv",
@@ -640,6 +689,9 @@ rule build_shipping_demand:
 
 
 rule build_transport_demand:
+    params:
+        snapshots=config["snapshots"],
+        sector=config["sector"],
     input:
         clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv",
         pop_weighted_energy_totals=RESOURCES
@@ -666,6 +718,19 @@ rule build_transport_demand:
 
 rule prepare_sector_network:
     params:
+        co2_budget=config["co2_budget"],
+        solving=config["solving"],
+        existing_capacities=config["existing_capacities"],
+        foresight=config["foresight"],
+        costs=config["costs"],
+        logging=config["logging"],
+        sector=config["sector"],
+        industry=config["industry"],
+        pypsa_eur=config["pypsa_eur"],
+        lines=config["lines"],
+        scenario=config["scenario"],
+        countries=config["countries"],
+        energy=config["energy"],
         RDIR=RDIR,
     input:
         **build_retro_cost_output,
diff --git a/rules/postprocess.smk b/rules/postprocess.smk
index fae0f856..1dfdd098 100644
--- a/rules/postprocess.smk
+++ b/rules/postprocess.smk
@@ -9,6 +9,10 @@ localrules:
 
 
 rule plot_network:
+    params:
+        logging=config["logging"],
+        foresight=config["foresight"],
+        plotting=config["plotting"],
     input:
         overrides="data/override_component_attrs",
         network=RESULTS
@@ -67,6 +71,11 @@ rule copy_conda_env:
 
 rule make_summary:
     params:
+        foresight=config["foresight"],
+        costs=config["costs"],
+        snapshots=config["snapshots"],
+        logging=config["logging"],
+        scenario=config["scenario"],
         RDIR=RDIR,
     input:
         overrides="data/override_component_attrs",
@@ -114,6 +123,10 @@ rule make_summary:
 
 rule plot_summary:
     params:
+        logging=config["logging"],
+        countries=config["countries"],
+        scenario=config["scenario"],
+        plotting=config["plotting"],
         RDIR=RDIR,
     input:
         costs=RESULTS + "csvs/costs.csv",
diff --git a/rules/retrieve.smk b/rules/retrieve.smk
index 0a96406a..4bfbd6c6 100644
--- a/rules/retrieve.smk
+++ b/rules/retrieve.smk
@@ -19,6 +19,8 @@ if config["enable"].get("retrieve_databundle", True):
         datafiles.extend(["natura/Natura2000_end2015.shp", "GEBCO_2014_2D.nc"])
 
     rule retrieve_databundle:
+        params:
+            tutorial=config["tutorial"],
         output:
             expand("data/bundle/{file}", file=datafiles),
         log:
diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk
index 8ddeca92..400220c1 100644
--- a/rules/solve_electricity.smk
+++ b/rules/solve_electricity.smk
@@ -4,6 +4,8 @@
 
 
 rule solve_network:
+    params:
+        solving=config["solving"],
     input:
         network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
     output:
@@ -30,6 +32,8 @@ rule solve_network:
 
 
 rule solve_operations_network:
+    params:
+        solving=config["solving"],
     input:
         network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
     output:
diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk
index f10d8157..2ecba999 100644
--- a/rules/solve_myopic.smk
+++ b/rules/solve_myopic.smk
@@ -4,6 +4,12 @@
 
 
 rule add_existing_baseyear:
+    params:
+        scenario=config["scenario"],
+        sector=config["sector"],
+        logging=config["logging"],
+        existing_capacities=config["existing_capacities"],
+        costs=config["costs"],
     input:
         overrides="data/override_component_attrs",
         network=RESULTS
@@ -42,6 +48,10 @@ rule add_existing_baseyear:
 
 
 rule add_brownfield:
+    params:
+        logging=config["logging"],
+        sector=config["sector"],
+        existing_capacities=config["existing_capacities"],
     input:
         overrides="data/override_component_attrs",
         network=RESULTS
@@ -74,6 +84,8 @@ ruleorder: add_existing_baseyear > add_brownfield
 
 
 rule solve_sector_network_myopic:
+    params:
+        solving=config["solving"],
     input:
         overrides="data/override_component_attrs",
         network=RESULTS
diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk
index c2e103e5..c3608471 100644
--- a/rules/solve_overnight.smk
+++ b/rules/solve_overnight.smk
@@ -4,6 +4,8 @@
 
 
 rule solve_sector_network:
+    params:
+        solving=config["solving"],
     input:
         overrides="data/override_component_attrs",
         network=RESULTS
diff --git a/scripts/_helpers.py b/scripts/_helpers.py
index a67fb105..2cc9a05a 100644
--- a/scripts/_helpers.py
+++ b/scripts/_helpers.py
@@ -82,7 +82,7 @@ def load_network(import_name=None, custom_components=None):
         As in pypsa.Network(import_name)
     custom_components : dict
         Dictionary listing custom components.
-        For using ``snakemake.config['override_components']``
+        For using ``snakemake.params['override_components']``
         in ``config/config.yaml`` define:
 
         .. code:: yaml
diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py
index e08b86d7..1d4e3a80 100644
--- a/scripts/add_brownfield.py
+++ b/scripts/add_brownfield.py
@@ -49,7 +49,7 @@ def add_brownfield(n, n_p, year):
             )
         ]
 
-        threshold = snakemake.config["existing_capacities"]["threshold_capacity"]
+        threshold = snakemake.params["existing_capacities"]["threshold_capacity"]
 
         if not chp_heat.empty:
             threshold_chp_heat = (
@@ -87,7 +87,7 @@ def add_brownfield(n, n_p, year):
 
         # deal with gas network
         pipe_carrier = ["gas pipeline"]
-        if snakemake.config["sector"]["H2_retrofit"]:
+        if snakemake.params["sector"]["H2_retrofit"]:
             # drop capacities of previous year to avoid duplicating
             to_drop = n.links.carrier.isin(pipe_carrier) & (n.links.build_year != year)
             n.mremove("Link", n.links.loc[to_drop].index)
@@ -98,7 +98,7 @@ def add_brownfield(n, n_p, year):
                 & (n.links.build_year != year)
             ].index
             gas_pipes_i = n.links[n.links.carrier.isin(pipe_carrier)].index
-            CH4_per_H2 = 1 / snakemake.config["sector"]["H2_retrofit_capacity_per_CH4"]
+            CH4_per_H2 = 1 / snakemake.params["sector"]["H2_retrofit_capacity_per_CH4"]
             fr = "H2 pipeline retrofitted"
             to = "gas pipeline"
             # today's pipe capacity
@@ -139,7 +139,7 @@ if __name__ == "__main__":
             planning_horizons=2030,
         )
 
-    logging.basicConfig(level=snakemake.config["logging"]["level"])
+    logging.basicConfig(level=snakemake.params["logging"]["level"])
 
     update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts)
 
diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index bed5ee6b..c3e847ce 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -686,15 +686,15 @@ def estimate_renewable_capacities(n, config):
             )
 
 
-def add_nice_carrier_names(n, plotting):
+def add_nice_carrier_names(n, config):
     carrier_i = n.carriers.index
     nice_names = (
-        pd.Series(plotting["nice_names"])
+        pd.Series(config["plotting"]["nice_names"])
         .reindex(carrier_i)
         .fillna(carrier_i.to_series().str.title())
     )
     n.carriers["nice_name"] = nice_names
-    colors = pd.Series(plotting["tech_colors"]).reindex(carrier_i)
+    colors = pd.Series(config["plotting"]["tech_colors"]).reindex(carrier_i)
     if colors.isna().any():
         missing_i = list(colors.index[colors.isna()])
         logger.warning(f"tech_colors for carriers {missing_i} not defined in config.")
@@ -713,23 +713,23 @@ if __name__ == "__main__":
 
     costs = load_costs(
         snakemake.input.tech_costs,
-        snakemake.param["costs"],
-        snakemake.param["electricity"],
+        snakemake.params["costs"],
+        snakemake.params["electricity"],
         Nyears,
     )
     ppl = load_powerplants(snakemake.input.powerplants)
 
-    if "renewable_carriers" in snakemake.param["electricity"]:
-        renewable_carriers = set(snakemake.param["electricity"]["renewable_carriers"])
+    if "renewable_carriers" in snakemake.params["electricity"]:
+        renewable_carriers = set(snakemake.params["electricity"]["renewable_carriers"])
     else:
         logger.warning(
             "Missing key `renewable_carriers` under config entry `electricity`. "
             "In future versions, this will raise an error. "
             "Falling back to carriers listed under `renewable`."
         )
-        renewable_carriers = snakemake.param["renewable"]
+        renewable_carriers = snakemake.params["renewable"]
 
-    extendable_carriers = snakemake.param["electricity"]["extendable_carriers"]
+    extendable_carriers = snakemake.params["electricity"]["extendable_carriers"]
     if not (set(renewable_carriers) & set(extendable_carriers["Generator"])):
         logger.warning(
             "No renewables found in config entry `extendable_carriers`. "
@@ -737,18 +737,18 @@ if __name__ == "__main__":
             "Falling back to all renewables."
         )
 
-    conventional_carriers = snakemake.param["electricity"]["conventional_carriers"]
+    conventional_carriers = snakemake.params["electricity"]["conventional_carriers"]
 
     attach_load(
         n,
         snakemake.input.regions,
         snakemake.input.load,
         snakemake.input.nuts3_shapes,
-        snakemake.param["countries"],
-        snakemake.param["load"]["scaling_factor"],
+        snakemake.params["countries"],
+        snakemake.params["load"]["scaling_factor"],
     )
 
-    update_transmission_costs(n, costs, snakemake.param["lines"]["length_factor"])
+    update_transmission_costs(n, costs, snakemake.params["lines"]["length_factor"])
 
     conventional_inputs = {
         k: v for k, v in snakemake.input.items() if k.startswith("conventional_")
@@ -759,7 +759,7 @@ if __name__ == "__main__":
         ppl,
         conventional_carriers,
         extendable_carriers,
-        snakemake.param.get("conventional", {}),
+        snakemake.config.get("conventional", {}),
         conventional_inputs,
     )
 
@@ -769,11 +769,11 @@ if __name__ == "__main__":
         snakemake.input,
         renewable_carriers,
         extendable_carriers,
-        snakemake.param["lines"]["length_factor"],
+        snakemake.params["lines"]["length_factor"],
     )
 
     if "hydro" in renewable_carriers:
-        conf = snakemake.param["renewable"]["hydro"]
+        conf = snakemake.params["renewable"]["hydro"]
         attach_hydro(
             n,
             costs,
@@ -784,7 +784,7 @@ if __name__ == "__main__":
             **conf,
         )
 
-    if "estimate_renewable_capacities" not in snakemake.param["electricity"]:
+    if "estimate_renewable_capacities" not in snakemake.params["electricity"]:
         logger.warning(
             "Missing key `estimate_renewable_capacities` under config entry `electricity`. "
             "In future versions, this will raise an error. "
@@ -792,18 +792,18 @@ if __name__ == "__main__":
         )
         if (
             "estimate_renewable_capacities_from_capacity_stats"
-            in snakemake.param["electricity"]
+            in snakemake.params["electricity"]
         ):
             estimate_renewable_caps = {
                 "enable": True,
-                **snakemake.param["electricity"][
+                **snakemake.params["electricity"][
                     "estimate_renewable_capacities_from_capacity_stats"
                 ],
             }
         else:
             estimate_renewable_caps = {"enable": False}
     else:
-        estimate_renewable_caps = snakemake.param["electricity"][
+        estimate_renewable_caps = snakemake.params["electricity"][
             "estimate_renewable_capacities"
         ]
     if "enable" not in estimate_renewable_caps:
@@ -819,21 +819,21 @@ if __name__ == "__main__":
             "Falling back to whether `renewable_capacities_from_opsd` is non-empty."
         )
         from_opsd = bool(
-            snakemake.param["electricity"].get("renewable_capacities_from_opsd", False)
+            snakemake.params["electricity"].get("renewable_capacities_from_opsd", False)
         )
         estimate_renewable_caps["from_opsd"] = from_opsd
 
     if estimate_renewable_caps["enable"]:
         if estimate_renewable_caps["from_opsd"]:
-            tech_map = snakemake.param["electricity"]["estimate_renewable_capacities"][
+            tech_map = snakemake.params["electricity"]["estimate_renewable_capacities"][
                 "technology_mapping"
             ]
             attach_OPSD_renewables(n, tech_map)
-        estimate_renewable_capacities(n, snakemake.param)
+        estimate_renewable_capacities(n, snakemake.config)
 
     update_p_nom_max(n)
 
-    add_nice_carrier_names(n, snakemake.param["plotting"])
+    add_nice_carrier_names(n, snakemake.config)
 
-    n.meta = snakemake.param
+    n.meta = snakemake.config
     n.export_to_netcdf(snakemake.output[0])
diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py
index 5bc0960d..a24b078d 100644
--- a/scripts/add_existing_baseyear.py
+++ b/scripts/add_existing_baseyear.py
@@ -157,7 +157,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas
     # Fill missing DateOut
     dateout = (
         df_agg.loc[biomass_i, "DateIn"]
-        + snakemake.config["costs"]["fill_values"]["lifetime"]
+        + snakemake.params["costs"]["fill_values"]["lifetime"]
     )
     df_agg.loc[biomass_i, "DateOut"] = df_agg.loc[biomass_i, "DateOut"].fillna(dateout)
 
@@ -218,7 +218,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas
         capacity = df.loc[grouping_year, generator]
         capacity = capacity[~capacity.isna()]
         capacity = capacity[
-            capacity > snakemake.config["existing_capacities"]["threshold_capacity"]
+            capacity > snakemake.params["existing_capacities"]["threshold_capacity"]
         ]
         suffix = "-ac" if generator == "offwind" else ""
         name_suffix = f" {generator}{suffix}-{grouping_year}"
@@ -582,7 +582,7 @@ def add_heating_capacities_installed_before_baseyear(
             )
 
             # delete links with capacities below threshold
-            threshold = snakemake.config["existing_capacities"]["threshold_capacity"]
+            threshold = snakemake.params["existing_capacities"]["threshold_capacity"]
             n.mremove(
                 "Link",
                 [
@@ -608,14 +608,14 @@ if __name__ == "__main__":
             planning_horizons=2020,
         )
 
-    logging.basicConfig(level=snakemake.config["logging"]["level"])
+    logging.basicConfig(level=snakemake.params["logging"]["level"])
 
     update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts)
 
-    options = snakemake.config["sector"]
+    options = snakemake.params["sector"]
     opts = snakemake.wildcards.sector_opts.split("-")
 
-    baseyear = snakemake.config["scenario"]["planning_horizons"][0]
+    baseyear = snakemake.params["scenario"]["planning_horizons"][0]
 
     overrides = override_component_attrs(snakemake.input.overrides)
     n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
@@ -626,14 +626,14 @@ if __name__ == "__main__":
     Nyears = n.snapshot_weightings.generators.sum() / 8760.0
     costs = prepare_costs(
         snakemake.input.costs,
-        snakemake.config["costs"],
+        snakemake.params["costs"],
         Nyears,
     )
 
-    grouping_years_power = snakemake.config["existing_capacities"][
+    grouping_years_power = snakemake.params["existing_capacities"][
         "grouping_years_power"
     ]
-    grouping_years_heat = snakemake.config["existing_capacities"]["grouping_years_heat"]
+    grouping_years_heat = snakemake.params["existing_capacities"]["grouping_years_heat"]
     add_power_capacities_installed_before_baseyear(
         n, grouping_years_power, costs, baseyear
     )
@@ -650,7 +650,7 @@ if __name__ == "__main__":
             .to_pandas()
             .reindex(index=n.snapshots)
         )
-        default_lifetime = snakemake.config["costs"]["fill_values"]["lifetime"]
+        default_lifetime = snakemake.params["costs"]["fill_values"]["lifetime"]
         add_heating_capacities_installed_before_baseyear(
             n,
             baseyear,
diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py
index 020370e5..08178c0a 100644
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@@ -235,11 +235,11 @@ if __name__ == "__main__":
     configure_logging(snakemake)
 
     n = pypsa.Network(snakemake.input.network)
-    elec_config = snakemake.config["electricity"]
+    elec_config = snakemake.params["electricity"]
 
     Nyears = n.snapshot_weightings.objective.sum() / 8760.0
     costs = load_costs(
-        snakemake.input.tech_costs, snakemake.config["costs"], elec_config, Nyears
+        snakemake.input.tech_costs, snakemake.params["costs"], elec_config, Nyears
     )
 
     attach_storageunits(n, costs, elec_config)
diff --git a/scripts/build_ammonia_production.py b/scripts/build_ammonia_production.py
index d78d627e..6f03324f 100644
--- a/scripts/build_ammonia_production.py
+++ b/scripts/build_ammonia_production.py
@@ -30,7 +30,7 @@ if __name__ == "__main__":
     ammonia.index = cc.convert(ammonia.index, to="iso2")
 
     years = [str(i) for i in range(2013, 2018)]
-    countries = ammonia.index.intersection(snakemake.config["countries"])
+    countries = ammonia.index.intersection(snakemake.params["countries"])
     ammonia = ammonia.loc[countries, years].astype(float)
 
     # convert from ktonN to ktonNH3
diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py
index 21d0e623..35218e2c 100644
--- a/scripts/build_biomass_potentials.py
+++ b/scripts/build_biomass_potentials.py
@@ -210,7 +210,7 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("build_biomass_potentials", simpl="", clusters="5")
 
-    config = snakemake.config["biomass"]
+    config = snakemake.params["biomass"]
     year = config["year"]
     scenario = config["scenario"]
 
diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py
index 6dc3b5a4..e9378792 100644
--- a/scripts/build_bus_regions.py
+++ b/scripts/build_bus_regions.py
@@ -116,7 +116,7 @@ if __name__ == "__main__":
         snakemake = mock_snakemake("build_bus_regions")
     configure_logging(snakemake)
 
-    countries = snakemake.config["countries"]
+    countries = snakemake.params["countries"]
 
     n = pypsa.Network(snakemake.input.base_network)
 
diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py
index 5d36cd5b..7128ec0d 100644
--- a/scripts/build_cop_profiles.py
+++ b/scripts/build_cop_profiles.py
@@ -39,7 +39,7 @@ if __name__ == "__main__":
         for source in ["air", "soil"]:
             source_T = xr.open_dataarray(snakemake.input[f"temp_{source}_{area}"])
 
-            delta_T = snakemake.config["sector"]["heat_pump_sink_T"] - source_T
+            delta_T = snakemake.params["sector"]["heat_pump_sink_T"] - source_T
 
             cop = coefficient_of_performance(delta_T, source)
 
diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py
index 365797d2..2f61f017 100644
--- a/scripts/build_cutout.py
+++ b/scripts/build_cutout.py
@@ -106,9 +106,9 @@ if __name__ == "__main__":
         snakemake = mock_snakemake("build_cutout", cutout="europe-2013-era5")
     configure_logging(snakemake)
 
-    cutout_params = snakemake.config["atlite"]["cutouts"][snakemake.wildcards.cutout]
+    cutout_params = snakemake.params["atlite"]["cutouts"][snakemake.wildcards.cutout]
 
-    snapshots = pd.date_range(freq="h", **snakemake.config["snapshots"])
+    snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"])
     time = [snapshots[0], snapshots[-1]]
     cutout_params["time"] = slice(*cutout_params.get("time", time))
 
diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py
index b86b4a5f..4ef56d1d 100755
--- a/scripts/build_electricity_demand.py
+++ b/scripts/build_electricity_demand.py
@@ -279,16 +279,16 @@ if __name__ == "__main__":
 
     configure_logging(snakemake)
 
-    powerstatistics = snakemake.config["load"]["power_statistics"]
-    interpolate_limit = snakemake.config["load"]["interpolate_limit"]
-    countries = snakemake.config["countries"]
-    snapshots = pd.date_range(freq="h", **snakemake.config["snapshots"])
+    powerstatistics = snakemake.params["load"]["power_statistics"]
+    interpolate_limit = snakemake.params["load"]["interpolate_limit"]
+    countries = snakemake.params["countries"]
+    snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"])
     years = slice(snapshots[0], snapshots[-1])
-    time_shift = snakemake.config["load"]["time_shift_for_large_gaps"]
+    time_shift = snakemake.params["load"]["time_shift_for_large_gaps"]
 
     load = load_timeseries(snakemake.input[0], years, countries, powerstatistics)
 
-    if snakemake.config["load"]["manual_adjustments"]:
+    if snakemake.params["load"]["manual_adjustments"]:
         load = manual_adjustment(load, snakemake.input[0], powerstatistics)
 
     logger.info(f"Linearly interpolate gaps of size {interpolate_limit} and less.")
diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py
index 45fc960f..6cedff97 100644
--- a/scripts/build_energy_totals.py
+++ b/scripts/build_energy_totals.py
@@ -373,7 +373,7 @@ def idees_per_country(ct, year, base_dir):
 
 def build_idees(countries, year):
     nprocesses = snakemake.threads
-    disable_progress = snakemake.config["run"].get("disable_progressbar", False)
+    disable_progress = snakemake.params["run"].get("disable_progressbar", False)
 
     func = partial(idees_per_country, year=year, base_dir=snakemake.input.idees)
     tqdm_kwargs = dict(
@@ -735,18 +735,18 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("build_energy_totals")
 
-    logging.basicConfig(level=snakemake.config["logging"]["level"])
+    logging.basicConfig(level=snakemake.params["logging"]["level"])
 
-    config = snakemake.config["energy"]
+    config = snakemake.params["energy"]
 
     nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index("index")
     population = nuts3["pop"].groupby(nuts3.country).sum()
 
-    countries = snakemake.config["countries"]
+    countries = snakemake.params["countries"]
     idees_countries = pd.Index(countries).intersection(eu28)
 
     data_year = config["energy_totals_year"]
-    report_year = snakemake.config["energy"]["eurostat_report_year"]
+    report_year = snakemake.params["energy"]["eurostat_report_year"]
     input_eurostat = snakemake.input.eurostat
     eurostat = build_eurostat(input_eurostat, countries, report_year, data_year)
     swiss = build_swiss(data_year)
@@ -756,7 +756,7 @@ if __name__ == "__main__":
     energy.to_csv(snakemake.output.energy_name)
 
     base_year_emissions = config["base_emissions_year"]
-    emissions_scope = snakemake.config["energy"]["emissions"]
+    emissions_scope = snakemake.params["energy"]["emissions"]
     eea_co2 = build_eea_co2(snakemake.input.co2, base_year_emissions, emissions_scope)
     eurostat_co2 = build_eurostat_co2(
         input_eurostat, countries, report_year, base_year_emissions
diff --git a/scripts/build_gas_input_locations.py b/scripts/build_gas_input_locations.py
index a3b945ab..59f31f5c 100644
--- a/scripts/build_gas_input_locations.py
+++ b/scripts/build_gas_input_locations.py
@@ -86,7 +86,7 @@ if __name__ == "__main__":
             clusters="37",
         )
 
-    logging.basicConfig(level=snakemake.config["logging"]["level"])
+    logging.basicConfig(level=snakemake.params["logging"]["level"])
 
     regions = load_bus_regions(
         snakemake.input.regions_onshore, snakemake.input.regions_offshore
diff --git a/scripts/build_gas_network.py b/scripts/build_gas_network.py
index 23f58caa..a70f15ff 100644
--- a/scripts/build_gas_network.py
+++ b/scripts/build_gas_network.py
@@ -147,7 +147,7 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("build_gas_network")
 
-    logging.basicConfig(level=snakemake.config["logging"]["level"])
+    logging.basicConfig(level=snakemake.params["logging"]["level"])
 
     gas_network = load_dataset(snakemake.input.gas_network)
 
diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py
index 56ceb4b7..655df28f 100644
--- a/scripts/build_heat_demand.py
+++ b/scripts/build_heat_demand.py
@@ -27,7 +27,7 @@ if __name__ == "__main__":
     cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1)
     client = Client(cluster, asynchronous=True)
 
-    time = pd.date_range(freq="h", **snakemake.config["snapshots"])
+    time = pd.date_range(freq="h", **snakemake.params["snapshots"])
     cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time)
 
     clustered_regions = (
diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py
index 0e8cfa27..5453ac5c 100644
--- a/scripts/build_hydro_profile.py
+++ b/scripts/build_hydro_profile.py
@@ -130,10 +130,10 @@ if __name__ == "__main__":
         snakemake = mock_snakemake("build_hydro_profile")
     configure_logging(snakemake)
 
-    config_hydro = snakemake.config["renewable"]["hydro"]
+    config_hydro = snakemake.params["renewable"]["hydro"]
     cutout = atlite.Cutout(snakemake.input.cutout)
 
-    countries = snakemake.config["countries"]
+    countries = snakemake.params["countries"]
     country_shapes = (
         gpd.read_file(snakemake.input.country_shapes)
         .set_index("name")["geometry"]
diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py
index 69daf64d..30c652d8 100644
--- a/scripts/build_industrial_distribution_key.py
+++ b/scripts/build_industrial_distribution_key.py
@@ -73,7 +73,7 @@ def prepare_hotmaps_database(regions):
 
     df[["srid", "coordinates"]] = df.geom.str.split(";", expand=True)
 
-    if snakemake.config["industry"].get("hotmaps_locate_missing", False):
+    if snakemake.params["industry"].get("hotmaps_locate_missing", False):
         df = locate_missing_industrial_sites(df)
 
     # remove those sites without valid locations
@@ -141,9 +141,9 @@ if __name__ == "__main__":
             clusters=48,
         )
 
-    logging.basicConfig(level=snakemake.config["logging"]["level"])
+    logging.basicConfig(level=snakemake.params["logging"]["level"])
 
-    countries = snakemake.config["countries"]
+    countries = snakemake.params["countries"]
 
     regions = gpd.read_file(snakemake.input.regions_onshore).set_index("name")
 
diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py
index 703997b1..4fec95cc 100644
--- a/scripts/build_industrial_energy_demand_per_country_today.py
+++ b/scripts/build_industrial_energy_demand_per_country_today.py
@@ -153,7 +153,7 @@ def add_non_eu28_industrial_energy_demand(countries, demand):
 
 def industrial_energy_demand(countries, year):
     nprocesses = snakemake.threads
-    disable_progress = snakemake.config["run"].get("disable_progressbar", False)
+    disable_progress = snakemake.params["run"].get("disable_progressbar", False)
     func = partial(
         industrial_energy_demand_per_country, year=year, jrc_dir=snakemake.input.jrc
     )
@@ -178,9 +178,9 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("build_industrial_energy_demand_per_country_today")
 
-    config = snakemake.config["industry"]
+    config = snakemake.params["industry"]
     year = config.get("reference_year", 2015)
-    countries = pd.Index(snakemake.config["countries"])
+    countries = pd.Index(snakemake.params["countries"])
 
     demand = industrial_energy_demand(countries.intersection(eu28), year)
 
diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py
index 437806b3..eb1b16cb 100644
--- a/scripts/build_industrial_production_per_country.py
+++ b/scripts/build_industrial_production_per_country.py
@@ -217,7 +217,7 @@ def industry_production_per_country(country, year, eurostat_dir, jrc_dir):
 
 def industry_production(countries, year, eurostat_dir, jrc_dir):
     nprocesses = snakemake.threads
-    disable_progress = snakemake.config["run"].get("disable_progressbar", False)
+    disable_progress = snakemake.params["run"].get("disable_progressbar", False)
 
     func = partial(
         industry_production_per_country,
@@ -277,13 +277,13 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("build_industrial_production_per_country")
 
-    logging.basicConfig(level=snakemake.config["logging"]["level"])
+    logging.basicConfig(level=snakemake.params["logging"]["level"])
 
-    countries = snakemake.config["countries"]
+    countries = snakemake.params["countries"]
 
-    year = snakemake.config["industry"]["reference_year"]
+    year = snakemake.params["industry"]["reference_year"]
 
-    config = snakemake.config["industry"]
+    config = snakemake.params["industry"]
 
     jrc_dir = snakemake.input.jrc
     eurostat_dir = snakemake.input.eurostat
diff --git a/scripts/build_industrial_production_per_country_tomorrow.py b/scripts/build_industrial_production_per_country_tomorrow.py
index 6c445608..b9ac9b16 100644
--- a/scripts/build_industrial_production_per_country_tomorrow.py
+++ b/scripts/build_industrial_production_per_country_tomorrow.py
@@ -15,7 +15,7 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("build_industrial_production_per_country_tomorrow")
 
-    config = snakemake.config["industry"]
+    config = snakemake.params["industry"]
 
     investment_year = int(snakemake.wildcards.planning_horizons)
 
diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py
index 54f2cfdc..c3e2bd49 100644
--- a/scripts/build_industry_sector_ratios.py
+++ b/scripts/build_industry_sector_ratios.py
@@ -439,7 +439,7 @@ def chemicals_industry():
 
     sector = "Ammonia"
     df[sector] = 0.0
-    if snakemake.config["sector"].get("ammonia", False):
+    if snakemake.params["sector"].get("ammonia", False):
         df.loc["ammonia", sector] = config["MWh_NH3_per_tNH3"]
     else:
         df.loc["hydrogen", sector] = config["MWh_H2_per_tNH3_electrolysis"]
@@ -1468,7 +1468,7 @@ if __name__ == "__main__":
     # TODO make config option
     year = 2015
 
-    config = snakemake.config["industry"]
+    config = snakemake.params["industry"]
 
     df = pd.concat(
         [
diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py
index e864d925..bc35dbcf 100644
--- a/scripts/build_population_layouts.py
+++ b/scripts/build_population_layouts.py
@@ -23,7 +23,7 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("build_population_layouts")
 
-    logging.basicConfig(level=snakemake.config["logging"]["level"])
+    logging.basicConfig(level=snakemake.params["logging"]["level"])
 
     cutout = atlite.Cutout(snakemake.input.cutout)
 
diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py
index 9ca67a53..bd0ee74e 100755
--- a/scripts/build_powerplants.py
+++ b/scripts/build_powerplants.py
@@ -115,7 +115,7 @@ if __name__ == "__main__":
     configure_logging(snakemake)
 
     n = pypsa.Network(snakemake.input.base_network)
-    countries = snakemake.config["countries"]
+    countries = snakemake.params["countries"]
 
     ppl = (
         pm.powerplants(from_url=True)
@@ -134,12 +134,12 @@ if __name__ == "__main__":
     ppl = ppl.query('not (Country in @available_countries and Fueltype == "Bioenergy")')
     ppl = pd.concat([ppl, opsd])
 
-    ppl_query = snakemake.config["electricity"]["powerplants_filter"]
+    ppl_query = snakemake.params["electricity"]["powerplants_filter"]
     if isinstance(ppl_query, str):
         ppl.query(ppl_query, inplace=True)
 
     # add carriers from own powerplant files:
-    custom_ppl_query = snakemake.config["electricity"]["custom_powerplants"]
+    custom_ppl_query = snakemake.params["electricity"]["custom_powerplants"]
     ppl = add_custom_powerplants(
         ppl, snakemake.input.custom_powerplants, custom_ppl_query
     )
diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py
index c0288aee..55541522 100644
--- a/scripts/build_renewable_profiles.py
+++ b/scripts/build_renewable_profiles.py
@@ -203,8 +203,8 @@ if __name__ == "__main__":
     configure_logging(snakemake)
 
     nprocesses = int(snakemake.threads)
-    noprogress = snakemake.config["run"].get("disable_progressbar", True)
-    config = snakemake.config["renewable"][snakemake.wildcards.technology]
+    noprogress = snakemake.params["run"].get("disable_progressbar", True)
+    config = snakemake.params["renewable"][snakemake.wildcards.technology]
     resource = config["resource"]  # pv panel config / wind turbine config
     correction_factor = config.get("correction_factor", 1.0)
     capacity_per_sqkm = config["capacity_per_sqkm"]
diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py
index 9dbfc375..c52d4eb6 100644
--- a/scripts/build_retro_cost.py
+++ b/scripts/build_retro_cost.py
@@ -305,7 +305,7 @@ def prepare_building_stock_data():
     u_values.set_index(["country_code", "subsector", "bage", "type"], inplace=True)
 
     #  only take in config.yaml specified countries into account
-    countries = snakemake.config["countries"]
+    countries = snakemake.params["countries"]
     area_tot = area_tot.loc[countries]
 
     return u_values, country_iso_dic, countries, area_tot, area
@@ -1040,7 +1040,7 @@ if __name__ == "__main__":
 
     #  ********  config  *********************************************************
 
-    retro_opts = snakemake.config["sector"]["retrofitting"]
+    retro_opts = snakemake.params["sector"]["retrofitting"]
     interest_rate = retro_opts["interest_rate"]
     annualise_cost = retro_opts["annualise_cost"]  # annualise the investment costs
     tax_weighting = retro_opts[
diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py
index 012effe8..5c388b2e 100644
--- a/scripts/build_sequestration_potentials.py
+++ b/scripts/build_sequestration_potentials.py
@@ -41,7 +41,7 @@ if __name__ == "__main__":
             "build_sequestration_potentials", simpl="", clusters="181"
         )
 
-    cf = snakemake.config["sector"]["regional_co2_sequestration_potential"]
+    cf = snakemake.params["sector"]["regional_co2_sequestration_potential"]
 
     gdf = gpd.read_file(snakemake.input.sequestration_potential[0])
 
diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py
index 50d21e12..f529517c 100644
--- a/scripts/build_shapes.py
+++ b/scripts/build_shapes.py
@@ -255,12 +255,12 @@ if __name__ == "__main__":
     configure_logging(snakemake)
 
     country_shapes = countries(
-        snakemake.input.naturalearth, snakemake.config["countries"]
+        snakemake.input.naturalearth, snakemake.params["countries"]
     )
     country_shapes.reset_index().to_file(snakemake.output.country_shapes)
 
     offshore_shapes = eez(
-        country_shapes, snakemake.input.eez, snakemake.config["countries"]
+        country_shapes, snakemake.input.eez, snakemake.params["countries"]
     )
     offshore_shapes.reset_index().to_file(snakemake.output.offshore_shapes)
 
diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py
index f4eb1557..180007b7 100644
--- a/scripts/build_solar_thermal_profiles.py
+++ b/scripts/build_solar_thermal_profiles.py
@@ -27,9 +27,9 @@ if __name__ == "__main__":
     cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1)
     client = Client(cluster, asynchronous=True)
 
-    config = snakemake.config["solar_thermal"]
+    config = snakemake.params["solar_thermal"]
 
-    time = pd.date_range(freq="h", **snakemake.config["snapshots"])
+    time = pd.date_range(freq="h", **snakemake.params["snapshots"])
     cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time)
 
     clustered_regions = (
diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py
index 8f6d6c6c..ee06aebb 100644
--- a/scripts/build_temperature_profiles.py
+++ b/scripts/build_temperature_profiles.py
@@ -27,7 +27,7 @@ if __name__ == "__main__":
     cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1)
     client = Client(cluster, asynchronous=True)
 
-    time = pd.date_range(freq="h", **snakemake.config["snapshots"])
+    time = pd.date_range(freq="h", **snakemake.params["snapshots"])
     cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time)
 
     clustered_regions = (
diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py
index 6b8bd04f..dc2b94b9 100644
--- a/scripts/build_transport_demand.py
+++ b/scripts/build_transport_demand.py
@@ -175,9 +175,9 @@ if __name__ == "__main__":
         snakemake.input.pop_weighted_energy_totals, index_col=0
     )
 
-    options = snakemake.config["sector"]
+    options = snakemake.params["sector"]
 
-    snapshots = pd.date_range(freq="h", **snakemake.config["snapshots"], tz="UTC")
+    snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"], tz="UTC")
 
     nyears = len(snapshots) / 8760
 
diff --git a/scripts/cluster_gas_network.py b/scripts/cluster_gas_network.py
index e7554dff..ba11ce1b 100755
--- a/scripts/cluster_gas_network.py
+++ b/scripts/cluster_gas_network.py
@@ -110,7 +110,7 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("cluster_gas_network", simpl="", clusters="37")
 
-    logging.basicConfig(level=snakemake.config["logging"]["level"])
+    logging.basicConfig(level=snakemake.params["logging"]["level"])
 
     fn = snakemake.input.cleaned_gas_network
     df = pd.read_csv(fn, index_col=0)
diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py
index 7572d3b3..78798804 100644
--- a/scripts/cluster_network.py
+++ b/scripts/cluster_network.py
@@ -468,18 +468,18 @@ if __name__ == "__main__":
         [
             tech
             for tech in n.generators.carrier.unique()
-            if tech in snakemake.config["renewable"]
+            if tech in snakemake.params["renewable"]
         ]
     )
 
-    exclude_carriers = snakemake.config["clustering"]["cluster_network"].get(
+    exclude_carriers = snakemake.params["clustering"]["cluster_network"].get(
         "exclude_carriers", []
     )
     aggregate_carriers = set(n.generators.carrier) - set(exclude_carriers)
     if snakemake.wildcards.clusters.endswith("m"):
         n_clusters = int(snakemake.wildcards.clusters[:-1])
         conventional = set(
-            snakemake.config["electricity"].get("conventional_carriers", [])
+            snakemake.params["electricity"].get("conventional_carriers", [])
         )
         aggregate_carriers = conventional.intersection(aggregate_carriers)
     elif snakemake.wildcards.clusters == "all":
@@ -495,13 +495,13 @@ if __name__ == "__main__":
             n, busmap, linemap, linemap, pd.Series(dtype="O")
         )
     else:
-        line_length_factor = snakemake.config["lines"]["length_factor"]
+        line_length_factor = snakemake.params["lines"]["length_factor"]
         Nyears = n.snapshot_weightings.objective.sum() / 8760
 
         hvac_overhead_cost = load_costs(
             snakemake.input.tech_costs,
-            snakemake.config["costs"],
-            snakemake.config["electricity"],
+            snakemake.params["costs"],
+            snakemake.params["electricity"],
             Nyears,
         ).at["HVAC overhead", "capital_cost"]
 
@@ -512,7 +512,7 @@ if __name__ == "__main__":
             ).all() or x.isnull().all(), "The `potential` configuration option must agree for all renewable carriers, for now!"
             return v
 
-        aggregation_strategies = snakemake.config["clustering"].get(
+        aggregation_strategies = snakemake.params["clustering"].get(
             "aggregation_strategies", {}
         )
         # translate str entries of aggregation_strategies to pd.Series functions:
@@ -521,7 +521,7 @@ if __name__ == "__main__":
             for p in aggregation_strategies.keys()
         }
 
-        custom_busmap = snakemake.config["enable"].get("custom_busmap", False)
+        custom_busmap = snakemake.params["enable"].get("custom_busmap", False)
         if custom_busmap:
             custom_busmap = pd.read_csv(
                 snakemake.input.custom_busmap, index_col=0, squeeze=True
@@ -539,7 +539,7 @@ if __name__ == "__main__":
             aggregate_carriers,
             line_length_factor,
             aggregation_strategies,
-            snakemake.config["solving"]["solver"]["name"],
+            snakemake.params["solving"]["solver"]["name"],
             cluster_config.get("algorithm", "hac"),
             cluster_config.get("feature", "solar+onwind-time"),
             hvac_overhead_cost,
diff --git a/scripts/make_summary.py b/scripts/make_summary.py
index 3d743942..56074525 100644
--- a/scripts/make_summary.py
+++ b/scripts/make_summary.py
@@ -198,7 +198,7 @@ def calculate_costs(n, label, costs):
 
 
 def calculate_cumulative_cost():
-    planning_horizons = snakemake.config["scenario"]["planning_horizons"]
+    planning_horizons = snakemake.params["scenario"]["planning_horizons"]
 
     cumulative_cost = pd.DataFrame(
         index=df["costs"].sum().index,
@@ -682,25 +682,25 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("make_summary")
 
-    logging.basicConfig(level=snakemake.config["logging"]["level"])
+    logging.basicConfig(level=snakemake.params["logging"]["level"])
 
     networks_dict = {
         (cluster, ll, opt + sector_opt, planning_horizon): "results/"
         + snakemake.params.RDIR
         + f"/postnetworks/elec_s{simpl}_{cluster}_l{ll}_{opt}_{sector_opt}_{planning_horizon}.nc"
-        for simpl in snakemake.config["scenario"]["simpl"]
-        for cluster in snakemake.config["scenario"]["clusters"]
-        for opt in snakemake.config["scenario"]["opts"]
-        for sector_opt in snakemake.config["scenario"]["sector_opts"]
-        for ll in snakemake.config["scenario"]["ll"]
-        for planning_horizon in snakemake.config["scenario"]["planning_horizons"]
+        for simpl in snakemake.params["scenario"]["simpl"]
+        for cluster in snakemake.params["scenario"]["clusters"]
+        for opt in snakemake.params["scenario"]["opts"]
+        for sector_opt in snakemake.params["scenario"]["sector_opts"]
+        for ll in snakemake.params["scenario"]["ll"]
+        for planning_horizon in snakemake.params["scenario"]["planning_horizons"]
     }
 
-    Nyears = len(pd.date_range(freq="h", **snakemake.config["snapshots"])) / 8760
+    Nyears = len(pd.date_range(freq="h", **snakemake.params["snapshots"])) / 8760
 
     costs_db = prepare_costs(
         snakemake.input.costs,
-        snakemake.config["costs"],
+        snakemake.params["costs"],
         Nyears,
     )
 
@@ -710,7 +710,7 @@ if __name__ == "__main__":
 
     to_csv(df)
 
-    if snakemake.config["foresight"] == "myopic":
+    if snakemake.params["foresight"] == "myopic":
         cumulative_cost = calculate_cumulative_cost()
         cumulative_cost.to_csv(
             "results/" + snakemake.params.RDIR + "/csvs/cumulative_cost.csv"
diff --git a/scripts/plot_network.py b/scripts/plot_network.py
index 0a22b2e5..8aac9db1 100644
--- a/scripts/plot_network.py
+++ b/scripts/plot_network.py
@@ -70,7 +70,7 @@ def plot_map(
     transmission=False,
     with_legend=True,
 ):
-    tech_colors = snakemake.config["plotting"]["tech_colors"]
+    tech_colors = snakemake.params["plotting"]["tech_colors"]
 
     n = network.copy()
     assign_location(n)
@@ -116,7 +116,7 @@ def plot_map(
     costs = costs.stack()  # .sort_index()
 
     # hack because impossible to drop buses...
-    eu_location = snakemake.config["plotting"].get(
+    eu_location = snakemake.params["plotting"].get(
         "eu_node_location", dict(x=-5.5, y=46)
     )
     n.buses.loc["EU gas", "x"] = eu_location["x"]
@@ -315,7 +315,7 @@ def plot_h2_map(network, regions):
     h2_new = n.links[n.links.carrier == "H2 pipeline"]
     h2_retro = n.links[n.links.carrier == "H2 pipeline retrofitted"]
 
-    if snakemake.config["foresight"] == "myopic":
+    if snakemake.params["foresight"] == "myopic":
         # sum capacitiy for pipelines from different investment periods
         h2_new = group_pipes(h2_new)
 
@@ -558,7 +558,7 @@ def plot_ch4_map(network):
     link_widths_used = max_usage / linewidth_factor
     link_widths_used[max_usage < line_lower_threshold] = 0.0
 
-    tech_colors = snakemake.config["plotting"]["tech_colors"]
+    tech_colors = snakemake.params["plotting"]["tech_colors"]
 
     pipe_colors = {
         "gas pipeline": "#f08080",
@@ -700,7 +700,7 @@ def plot_map_without(network):
 
     # hack because impossible to drop buses...
     if "EU gas" in n.buses.index:
-        eu_location = snakemake.config["plotting"].get(
+        eu_location = snakemake.params["plotting"].get(
             "eu_node_location", dict(x=-5.5, y=46)
         )
         n.buses.loc["EU gas", "x"] = eu_location["x"]
@@ -876,7 +876,7 @@ def plot_series(network, carrier="AC", name="test"):
             stacked=True,
             linewidth=0.0,
             color=[
-                snakemake.config["plotting"]["tech_colors"][i.replace(suffix, "")]
+                snakemake.params["plotting"]["tech_colors"][i.replace(suffix, "")]
                 for i in new_columns
             ],
         )
@@ -930,14 +930,14 @@ if __name__ == "__main__":
             planning_horizons="2030",
         )
 
-    logging.basicConfig(level=snakemake.config["logging"]["level"])
+    logging.basicConfig(level=snakemake.params["logging"]["level"])
 
     overrides = override_component_attrs(snakemake.input.overrides)
     n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
 
     regions = gpd.read_file(snakemake.input.regions).set_index("name")
 
-    map_opts = snakemake.config["plotting"]["map"]
+    map_opts = snakemake.params["plotting"]["map"]
 
     if map_opts["boundaries"] is None:
         map_opts["boundaries"] = regions.total_bounds[[0, 2, 1, 3]] + [-1, 1, -1, 1]
diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py
index cfa8e361..e18ff61b 100644
--- a/scripts/plot_summary.py
+++ b/scripts/plot_summary.py
@@ -142,10 +142,10 @@ def plot_costs():
 
     df = df.groupby(df.index.map(rename_techs)).sum()
 
-    to_drop = df.index[df.max(axis=1) < snakemake.config["plotting"]["costs_threshold"]]
+    to_drop = df.index[df.max(axis=1) < snakemake.params["plotting"]["costs_threshold"]]
 
     logger.info(
-        f"Dropping technology with costs below {snakemake.config['plotting']['costs_threshold']} EUR billion per year"
+        f"Dropping technology with costs below {snakemake.params['plotting']['costs_threshold']} EUR billion per year"
     )
     logger.debug(df.loc[to_drop])
 
@@ -165,7 +165,7 @@ def plot_costs():
         kind="bar",
         ax=ax,
         stacked=True,
-        color=[snakemake.config["plotting"]["tech_colors"][i] for i in new_index],
+        color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index],
     )
 
     handles, labels = ax.get_legend_handles_labels()
@@ -173,7 +173,7 @@ def plot_costs():
     handles.reverse()
     labels.reverse()
 
-    ax.set_ylim([0, snakemake.config["plotting"]["costs_max"]])
+    ax.set_ylim([0, snakemake.params["plotting"]["costs_max"]])
 
     ax.set_ylabel("System Cost [EUR billion per year]")
 
@@ -201,11 +201,11 @@ def plot_energy():
     df = df.groupby(df.index.map(rename_techs)).sum()
 
     to_drop = df.index[
-        df.abs().max(axis=1) < snakemake.config["plotting"]["energy_threshold"]
+        df.abs().max(axis=1) < snakemake.params["plotting"]["energy_threshold"]
     ]
 
     logger.info(
-        f"Dropping all technology with energy consumption or production below {snakemake.config['plotting']['energy_threshold']} TWh/a"
+        f"Dropping all technology with energy consumption or production below {snakemake.params['plotting']['energy_threshold']} TWh/a"
     )
     logger.debug(df.loc[to_drop])
 
@@ -227,7 +227,7 @@ def plot_energy():
         kind="bar",
         ax=ax,
         stacked=True,
-        color=[snakemake.config["plotting"]["tech_colors"][i] for i in new_index],
+        color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index],
     )
 
     handles, labels = ax.get_legend_handles_labels()
@@ -237,8 +237,8 @@ def plot_energy():
 
     ax.set_ylim(
         [
-            snakemake.config["plotting"]["energy_min"],
-            snakemake.config["plotting"]["energy_max"],
+            snakemake.params["plotting"]["energy_min"],
+            snakemake.params["plotting"]["energy_max"],
         ]
     )
 
@@ -287,7 +287,7 @@ def plot_balances():
         df = df.groupby(df.index.map(rename_techs)).sum()
 
         to_drop = df.index[
-            df.abs().max(axis=1) < snakemake.config["plotting"]["energy_threshold"] / 10
+            df.abs().max(axis=1) < snakemake.params["plotting"]["energy_threshold"] / 10
         ]
 
         if v[0] in co2_carriers:
@@ -296,7 +296,7 @@ def plot_balances():
             units = "TWh/a"
 
         logger.debug(
-            f"Dropping technology energy balance smaller than {snakemake.config['plotting']['energy_threshold']/10} {units}"
+            f"Dropping technology energy balance smaller than {snakemake.params['plotting']['energy_threshold']/10} {units}"
         )
         logger.debug(df.loc[to_drop])
 
@@ -317,7 +317,7 @@ def plot_balances():
             kind="bar",
             ax=ax,
             stacked=True,
-            color=[snakemake.config["plotting"]["tech_colors"][i] for i in new_index],
+            color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index],
         )
 
         handles, labels = ax.get_legend_handles_labels()
@@ -455,10 +455,10 @@ def plot_carbon_budget_distribution(input_eurostat):
     ax1 = plt.subplot(gs1[0, 0])
     ax1.set_ylabel("CO$_2$ emissions (Gt per year)", fontsize=22)
     ax1.set_ylim([0, 5])
-    ax1.set_xlim([1990, snakemake.config["scenario"]["planning_horizons"][-1] + 1])
+    ax1.set_xlim([1990, snakemake.params["scenario"]["planning_horizons"][-1] + 1])
 
     path_cb = "results/" + snakemake.params.RDIR + "/csvs/"
-    countries = snakemake.config["countries"]
+    countries = snakemake.params["countries"]
     e_1990 = co2_emissions_year(countries, input_eurostat, opts, year=1990)
     CO2_CAP = pd.read_csv(path_cb + "carbon_budget_distribution.csv", index_col=0)
 
@@ -545,7 +545,7 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("plot_summary")
 
-    logging.basicConfig(level=snakemake.config["logging"]["level"])
+    logging.basicConfig(level=snakemake.params["logging"]["level"])
 
     n_header = 4
 
@@ -555,7 +555,7 @@ if __name__ == "__main__":
 
     plot_balances()
 
-    for sector_opts in snakemake.config["scenario"]["sector_opts"]:
+    for sector_opts in snakemake.params["scenario"]["sector_opts"]:
         opts = sector_opts.split("-")
         for o in opts:
             if "cb" in o:
diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index 14a003ae..46e4e74a 100755
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -253,12 +253,12 @@ if __name__ == "__main__":
     Nyears = n.snapshot_weightings.objective.sum() / 8760.0
     costs = load_costs(
         snakemake.input.tech_costs,
-        snakemake.config["costs"],
-        snakemake.config["electricity"],
+        snakemake.params["costs"],
+        snakemake.params["electricity"],
         Nyears,
     )
 
-    set_line_s_max_pu(n, snakemake.config["lines"]["s_max_pu"])
+    set_line_s_max_pu(n, snakemake.params["lines"]["s_max_pu"])
 
     for o in opts:
         m = re.match(r"^\d+h$", o, re.IGNORECASE)
@@ -269,7 +269,7 @@ if __name__ == "__main__":
     for o in opts:
         m = re.match(r"^\d+seg$", o, re.IGNORECASE)
         if m is not None:
-            solver_name = snakemake.config["solving"]["solver"]["name"]
+            solver_name = snakemake.params["solving"]["solver"]["name"]
             n = apply_time_segmentation(n, m.group(0)[:-3], solver_name)
             break
 
@@ -277,11 +277,11 @@ if __name__ == "__main__":
         if "Co2L" in o:
             m = re.findall("[0-9]*\.?[0-9]+$", o)
             if len(m) > 0:
-                co2limit = float(m[0]) * snakemake.config["electricity"]["co2base"]
+                co2limit = float(m[0]) * snakemake.params["electricity"]["co2base"]
                 add_co2limit(n, co2limit, Nyears)
                 logger.info("Setting CO2 limit according to wildcard value.")
             else:
-                add_co2limit(n, snakemake.config["electricity"]["co2limit"], Nyears)
+                add_co2limit(n, snakemake.params["electricity"]["co2limit"], Nyears)
                 logger.info("Setting CO2 limit according to config value.")
             break
 
@@ -293,7 +293,7 @@ if __name__ == "__main__":
                 add_gaslimit(n, limit, Nyears)
                 logger.info("Setting gas usage limit according to wildcard value.")
             else:
-                add_gaslimit(n, snakemake.config["electricity"].get("gaslimit"), Nyears)
+                add_gaslimit(n, snakemake.params["electricity"].get("gaslimit"), Nyears)
                 logger.info("Setting gas usage limit according to config value.")
             break
 
@@ -322,7 +322,7 @@ if __name__ == "__main__":
                 add_emission_prices(n, dict(co2=float(m[0])))
             else:
                 logger.info("Setting emission prices according to config value.")
-                add_emission_prices(n, snakemake.config["costs"]["emission_prices"])
+                add_emission_prices(n, snakemake.params["costs"]["emission_prices"])
             break
 
     ll_type, factor = snakemake.wildcards.ll[0], snakemake.wildcards.ll[1:]
@@ -330,8 +330,8 @@ if __name__ == "__main__":
 
     set_line_nom_max(
         n,
-        s_nom_max_set=snakemake.config["lines"].get("s_nom_max,", np.inf),
-        p_nom_max_set=snakemake.config["links"].get("p_nom_max,", np.inf),
+        s_nom_max_set=snakemake.params["lines"].get("s_nom_max,", np.inf),
+        p_nom_max_set=snakemake.params["links"].get("p_nom_max,", np.inf),
     )
 
     if "ATK" in opts:
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 1cb7146e..c69f7290 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -200,12 +200,12 @@ def co2_emissions_year(
     """
     Calculate CO2 emissions in one specific year (e.g. 1990 or 2018).
     """
-    emissions_scope = snakemake.config["energy"]["emissions"]
+    emissions_scope = snakemake.params["energy"]["emissions"]
     eea_co2 = build_eea_co2(snakemake.input.co2, year, emissions_scope)
 
     # TODO: read Eurostat data from year > 2014
     # this only affects the estimation of CO2 emissions for BA, RS, AL, ME, MK
-    report_year = snakemake.config["energy"]["eurostat_report_year"]
+    report_year = snakemake.params["energy"]["eurostat_report_year"]
     if year > 2014:
         eurostat_co2 = build_eurostat_co2(
             input_eurostat, countries, report_year, year=2014
@@ -241,7 +241,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year):
         carbon_budget = float(o[o.find("cb") + 2 : o.find("ex")])
         r = float(o[o.find("ex") + 2 :])
 
-    countries = snakemake.config["countries"]
+    countries = snakemake.params["countries"]
 
     e_1990 = co2_emissions_year(
         countries, input_eurostat, opts, emissions_scope, report_year, year=1990
@@ -252,7 +252,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year):
         countries, input_eurostat, opts, emissions_scope, report_year, year=2018
     )
 
-    planning_horizons = snakemake.config["scenario"]["planning_horizons"]
+    planning_horizons = snakemake.params["scenario"]["planning_horizons"]
     t_0 = planning_horizons[0]
 
     if "be" in o:
@@ -391,7 +391,7 @@ def update_wind_solar_costs(n, costs):
         with xr.open_dataset(profile) as ds:
             underwater_fraction = ds["underwater_fraction"].to_pandas()
             connection_cost = (
-                snakemake.config["lines"]["length_factor"]
+                snakemake.params["lines"]["length_factor"]
                 * ds["average_distance"].to_pandas()
                 * (
                     underwater_fraction
@@ -483,8 +483,8 @@ def remove_elec_base_techs(n):
     batteries and H2) from base electricity-only network, since they're added
     here differently using links.
     """
-    for c in n.iterate_components(snakemake.config["pypsa_eur"]):
-        to_keep = snakemake.config["pypsa_eur"][c.name]
+    for c in n.iterate_components(snakemake.params["pypsa_eur"]):
+        to_keep = snakemake.params["pypsa_eur"][c.name]
         to_remove = pd.Index(c.df.carrier.unique()).symmetric_difference(to_keep)
         if to_remove.empty:
             continue
@@ -674,7 +674,7 @@ def add_dac(n, costs):
 def add_co2limit(n, nyears=1.0, limit=0.0):
     logger.info(f"Adding CO2 budget limit as per unit of 1990 levels of {limit}")
 
-    countries = snakemake.config["countries"]
+    countries = snakemake.params["countries"]
 
     sectors = emission_sectors_from_opts(opts)
 
@@ -787,7 +787,7 @@ def add_ammonia(n, costs):
 
     nodes = pop_layout.index
 
-    cf_industry = snakemake.config["industry"]
+    cf_industry = snakemake.params["industry"]
 
     n.add("Carrier", "NH3")
 
@@ -1102,7 +1102,7 @@ def add_storage_and_grids(n, costs):
             lifetime=costs.at["OCGT", "lifetime"],
         )
 
-    cavern_types = snakemake.config["sector"]["hydrogen_underground_storage_locations"]
+    cavern_types = snakemake.params["sector"]["hydrogen_underground_storage_locations"]
     h2_caverns = pd.read_csv(snakemake.input.h2_cavern, index_col=0)
 
     if not h2_caverns.empty and options["hydrogen_underground_storage"]:
@@ -3266,11 +3266,11 @@ if __name__ == "__main__":
             planning_horizons="2030",
         )
 
-    logging.basicConfig(level=snakemake.config["logging"]["level"])
+    logging.basicConfig(level=snakemake.params["logging"]["level"])
 
     update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts)
 
-    options = snakemake.config["sector"]
+    options = snakemake.params["sector"]
 
     opts = snakemake.wildcards.sector_opts.split("-")
 
@@ -3285,7 +3285,7 @@ if __name__ == "__main__":
 
     costs = prepare_costs(
         snakemake.input.costs,
-        snakemake.config["costs"],
+        snakemake.params["costs"],
         nyears,
     )
 
@@ -3297,10 +3297,10 @@ if __name__ == "__main__":
 
     spatial = define_spatial(pop_layout.index, options)
 
-    if snakemake.config["foresight"] == "myopic":
+    if snakemake.params["foresight"] == "myopic":
         add_lifetime_wind_solar(n, costs)
 
-        conventional = snakemake.config["existing_capacities"]["conventional_carriers"]
+        conventional = snakemake.params["existing_capacities"]["conventional_carriers"]
         for carrier in conventional:
             add_carrier_buses(n, carrier)
 
@@ -3365,19 +3365,19 @@ if __name__ == "__main__":
     if options["allam_cycle"]:
         add_allam(n, costs)
 
-    solver_name = snakemake.config["solving"]["solver"]["name"]
+    solver_name = snakemake.params["solving"]["solver"]["name"]
     n = set_temporal_aggregation(n, opts, solver_name)
 
     limit_type = "config"
-    limit = get(snakemake.config["co2_budget"], investment_year)
+    limit = get(snakemake.params["co2_budget"], investment_year)
     for o in opts:
         if "cb" not in o:
             continue
         limit_type = "carbon budget"
         fn = "results/" + snakemake.params.RDIR + "/csvs/carbon_budget_distribution.csv"
         if not os.path.exists(fn):
-            emissions_scope = snakemake.config["energy"]["emissions"]
-            report_year = snakemake.config["energy"]["eurostat_report_year"]
+            emissions_scope = snakemake.params["energy"]["emissions"]
+            report_year = snakemake.params["energy"]["eurostat_report_year"]
             build_carbon_budget(
                 o, snakemake.input.eurostat, fn, emissions_scope, report_year
             )
@@ -3412,8 +3412,8 @@ if __name__ == "__main__":
     if options["electricity_grid_connection"]:
         add_electricity_grid_connection(n, costs)
 
-    first_year_myopic = (snakemake.config["foresight"] == "myopic") and (
-        snakemake.config["scenario"]["planning_horizons"][0] == investment_year
+    first_year_myopic = (snakemake.params["foresight"] == "myopic") and (
+        snakemake.params["scenario"]["planning_horizons"][0] == investment_year
     )
 
     if options.get("cluster_heat_buses", False) and not first_year_myopic:
diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py
index 0c6a7feb..0271166c 100644
--- a/scripts/retrieve_databundle.py
+++ b/scripts/retrieve_databundle.py
@@ -53,7 +53,7 @@ if __name__ == "__main__":
         snakemake
     )  # TODO Make logging compatible with progressbar (see PR #102)
 
-    if snakemake.config["tutorial"]:
+    if snakemake.params["tutorial"]:
         url = "https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz"
     else:
         url = "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz"
@@ -63,7 +63,7 @@ if __name__ == "__main__":
     to_fn = Path(f"{rootpath}/data")
 
     logger.info(f"Downloading databundle from '{url}'.")
-    disable_progress = snakemake.config["run"].get("disable_progressbar", False)
+    disable_progress = snakemake.params["run"].get("disable_progressbar", False)
     progress_retrieve(url, tarball_fn, disable=disable_progress)
 
     logger.info("Extracting databundle.")
diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 2be8c36a..bb7f615c 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -510,7 +510,7 @@ if __name__ == "__main__":
 
     n = pypsa.Network(snakemake.input.network)
 
-    aggregation_strategies = snakemake.config["clustering"].get(
+    aggregation_strategies = snakemake.params["clustering"].get(
         "aggregation_strategies", {}
     )
     # translate str entries of aggregation_strategies to pd.Series functions:
@@ -525,8 +525,8 @@ if __name__ == "__main__":
 
     technology_costs = load_costs(
         snakemake.input.tech_costs,
-        snakemake.config["costs"],
-        snakemake.config["electricity"],
+        snakemake.params["costs"],
+        snakemake.params["electricity"],
         Nyears,
     )
 
@@ -536,7 +536,7 @@ if __name__ == "__main__":
 
     busmaps = [trafo_map, simplify_links_map]
 
-    cluster_config = snakemake.config["clustering"]["simplify_network"]
+    cluster_config = snakemake.params["clustering"]["simplify_network"]
     if cluster_config.get("remove_stubs", True):
         n, stub_map = remove_stubs(
             n,
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index ff1c0ccf..c7041e85 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -663,7 +663,7 @@ if __name__ == "__main__":
     if "sector_opts" in snakemake.wildcards.keys():
         opts += "-" + snakemake.wildcards.sector_opts
     opts = [o for o in opts.split("-") if o != ""]
-    solve_opts = snakemake.config["solving"]["options"]
+    solve_opts = snakemake.params["solving"]["options"]
 
     np.random.seed(solve_opts.get("seed", 123))
 
diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py
index 25fe0753..c1b2be6f 100644
--- a/scripts/solve_operations_network.py
+++ b/scripts/solve_operations_network.py
@@ -42,7 +42,7 @@ if __name__ == "__main__":
 
     opts = (snakemake.wildcards.opts + "-" + snakemake.wildcards.sector_opts).split("-")
     opts = [o for o in opts if o != ""]
-    solve_opts = snakemake.config["solving"]["options"]
+    solve_opts = snakemake.params["solving"]["options"]
 
     np.random.seed(solve_opts.get("seed", 123))
 

From 61893c3c9b9e9f738d795f31ab03be07f6558ebd Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Mon, 15 May 2023 11:26:36 +0200
Subject: [PATCH 097/176] fixing build_electricity.smk and retrieve.smk

---
 rules/build_electricity.smk | 1 +
 rules/retrieve.smk          | 1 +
 2 files changed, 2 insertions(+)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index bb000f2b..56098ddd 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -201,6 +201,7 @@ rule build_ship_raster:
 
 rule build_renewable_profiles:
     params:
+        run=config["run"],
         renewable=config["renewable"],
     input:
         base_network=RESOURCES + "networks/base.nc",
diff --git a/rules/retrieve.smk b/rules/retrieve.smk
index 4bfbd6c6..bc8756d5 100644
--- a/rules/retrieve.smk
+++ b/rules/retrieve.smk
@@ -20,6 +20,7 @@ if config["enable"].get("retrieve_databundle", True):
 
     rule retrieve_databundle:
         params:
+            run=config["run"],
             tutorial=config["tutorial"],
         output:
             expand("data/bundle/{file}", file=datafiles),

From e431a9675cd7da4b229efe95f92ba44e73ff87a9 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 15 May 2023 13:02:39 +0000
Subject: [PATCH 098/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 doc/release_notes.rst             | 4 ++--
 scripts/prepare_sector_network.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index be06170a..06e2891e 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -32,8 +32,8 @@ Upcoming Release
 * Add option to include a piecewise linear approximation of transmission losses,
   e.g. by setting ``solving: options: transmission_losses: 2`` for an
   approximation with two tangents.
-  
-  
+
+
 PyPSA-Eur 0.8.0 (18th March 2023)
 =================================
 
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index c19011b8..4fe9f389 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -22,7 +22,7 @@ from _helpers import (
     override_component_attrs,
     update_config_with_sector_opts,
 )
-from add_electricity import sanitize_carriers, calculate_annuity
+from add_electricity import calculate_annuity, sanitize_carriers
 from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2
 from networkx.algorithms import complement
 from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation

From 748cfaec65aeaa1b4a671ac854aaa46cfd0941e7 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 15 May 2023 22:01:44 +0000
Subject: [PATCH 099/176] [pre-commit.ci] pre-commit autoupdate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

updates:
- [github.com/macisamuele/language-formatters-pre-commit-hooks: v2.8.0 → v2.9.0](https://github.com/macisamuele/language-formatters-pre-commit-hooks/compare/v2.8.0...v2.9.0)
---
 .pre-commit-config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 88685f3b..48915f4c 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -67,7 +67,7 @@ repos:
 
   # Do YAML formatting (before the linter checks it for misses)
 - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
-  rev: v2.8.0
+  rev: v2.9.0
   hooks:
   - id: pretty-format-yaml
     args: [--autofix, --indent, "2", --preserve-quotes]

From 875fcdd0e2ee1ef85e9d769446f1ebf7e836b4e0 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Tue, 16 May 2023 13:34:23 +0200
Subject: [PATCH 100/176] address geopandas future warning

---
 scripts/build_shapes.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py
index 50d21e12..54a7774e 100644
--- a/scripts/build_shapes.py
+++ b/scripts/build_shapes.py
@@ -234,6 +234,7 @@ def nuts3(country_shapes, nuts3, nuts3pop, nuts3gdp, ch_cantons, ch_popgdp):
     manual = gpd.GeoDataFrame(
         [["BA1", "BA", 3871.0], ["RS1", "RS", 7210.0], ["AL1", "AL", 2893.0]],
         columns=["NUTS_ID", "country", "pop"],
+        geometry=gpd.GeoSeries()
     )
     manual["geometry"] = manual["country"].map(country_shapes)
     manual = manual.dropna()

From 7d4dacf8bdeb24513888719065069102152b9be4 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 16 May 2023 11:34:58 +0000
Subject: [PATCH 101/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/build_shapes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py
index 54a7774e..5561cfb2 100644
--- a/scripts/build_shapes.py
+++ b/scripts/build_shapes.py
@@ -234,7 +234,7 @@ def nuts3(country_shapes, nuts3, nuts3pop, nuts3gdp, ch_cantons, ch_popgdp):
     manual = gpd.GeoDataFrame(
         [["BA1", "BA", 3871.0], ["RS1", "RS", 7210.0], ["AL1", "AL", 2893.0]],
         columns=["NUTS_ID", "country", "pop"],
-        geometry=gpd.GeoSeries()
+        geometry=gpd.GeoSeries(),
     )
     manual["geometry"] = manual["country"].map(country_shapes)
     manual = manual.dropna()

From 7fad9e263569db7eaf392c56e47e01c552343342 Mon Sep 17 00:00:00 2001
From: Lissy Langer <54096244+lilanger@users.noreply.github.com>
Date: Tue, 16 May 2023 14:57:45 +0200
Subject: [PATCH 102/176] Update supply_demand.rst

Heat pump COP equations signs are wrong: https://pubs.rsc.org/en/content/articlelanding/2012/EE/c2ee22653g
---
 doc/supply_demand.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/supply_demand.rst b/doc/supply_demand.rst
index 16242405..b043268b 100644
--- a/doc/supply_demand.rst
+++ b/doc/supply_demand.rst
@@ -133,12 +133,12 @@ The coefficient of performance (COP) of air- and ground-sourced heat pumps depen
 For the sink water temperature Tsink we assume 55 °C [`Config <https://github.com/PyPSA/pypsa-eur-sec/blob/3daff49c9999ba7ca7534df4e587e1d516044fc3/config.default.yaml#L207>`_ file]. For the time- and location-dependent source temperatures Tsource, we rely on the `ERA5 <https://doi.org/10.1002/qj.3803>`_ reanalysis weather data. The temperature differences are converted into COP time series using results from a regression analysis performed in the study by `Stafell et al. <https://pubs.rsc.org/en/content/articlelanding/2012/EE/c2ee22653g>`_. For air-sourced heat pumps (ASHP), we use the function:
 
 .. math::
-   COP (\Delta T) = 6.81 + 0.121\Delta T + 0.000630\Delta T^2
+   COP (\Delta T) = 6.81 - 0.121\Delta T + 0.000630\Delta T^2
 
 for ground-sourced heat pumps (GSHP), we use the function:
 
 .. math::
-   COP(\Delta T) = 8.77 + 0.150\Delta T + 0.000734\Delta T^2
+   COP(\Delta T) = 8.77 - 0.150\Delta T + 0.000734\Delta T^2
 
 **Resistive heaters**
 

From 8d6d6610712e1f5a2ce3bfab13a1a23afec21029 Mon Sep 17 00:00:00 2001
From: Max Parzen <max.parzen@ed.ac.uk>
Date: Tue, 16 May 2023 20:44:22 +0100
Subject: [PATCH 103/176] Update support.rst

---
 doc/support.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/support.rst b/doc/support.rst
index 36d1a2dd..1d512d59 100644
--- a/doc/support.rst
+++ b/doc/support.rst
@@ -9,6 +9,6 @@ Support
 
 * In case of code-related **questions**, please post on `stack overflow <https://stackoverflow.com/questions/tagged/pypsa>`_.
 * For non-programming related and more general questions please refer to the `mailing list <https://groups.google.com/group/pypsa>`_.
-* To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.gg/JTdvaEBb).
+* To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the `discord server <https://discord.gg/AnuJBk23FU>`_.
 * For **bugs and feature requests**, please use the `issue tracker <https://github.com/PyPSA/pypsa-eur/issues>`_.
 * We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github <https://github.com/PyPSA/PyPSA>`_. For further information on how to contribute, please refer to :ref:`contributing`.

From 8208ac033baff982fbeedb11ef019d4bd581576c Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Wed, 17 May 2023 18:43:30 +0200
Subject: [PATCH 104/176] convert param back to config for "logging" and "run",
 made "enable" more specific

---
 rules/build_electricity.smk                       |  3 +--
 rules/build_sector.smk                            | 15 ---------------
 rules/postprocess.smk                             |  3 ---
 rules/retrieve.smk                                |  1 -
 rules/solve_myopic.smk                            |  2 --
 scripts/add_brownfield.py                         |  2 +-
 scripts/add_existing_baseyear.py                  |  2 +-
 scripts/build_energy_totals.py                    |  4 ++--
 scripts/build_gas_input_locations.py              |  2 +-
 scripts/build_gas_network.py                      |  2 +-
 scripts/build_industrial_distribution_key.py      |  2 +-
 ..._industrial_energy_demand_per_country_today.py |  2 +-
 .../build_industrial_production_per_country.py    |  4 ++--
 scripts/build_population_layouts.py               |  2 +-
 scripts/build_renewable_profiles.py               |  2 +-
 scripts/cluster_gas_network.py                    |  2 +-
 scripts/make_summary.py                           |  2 +-
 scripts/plot_network.py                           |  2 +-
 scripts/plot_summary.py                           |  2 +-
 scripts/prepare_sector_network.py                 |  2 +-
 scripts/retrieve_databundle.py                    |  2 +-
 21 files changed, 19 insertions(+), 41 deletions(-)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index 56098ddd..9414c25b 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -201,7 +201,6 @@ rule build_ship_raster:
 
 rule build_renewable_profiles:
     params:
-        run=config["run"],
         renewable=config["renewable"],
     input:
         base_network=RESOURCES + "networks/base.nc",
@@ -350,7 +349,7 @@ rule cluster_network:
         lines=config["lines"],
         renewable=config["renewable"],
         clustering=config["clustering"],
-        enable=config["enable"],
+        enable=config["enable"].get("custom_busmap", False),
     input:
         network=RESOURCES + "networks/elec_s{simpl}.nc",
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson",
diff --git a/rules/build_sector.smk b/rules/build_sector.smk
index d375b7b9..9faae4b9 100644
--- a/rules/build_sector.smk
+++ b/rules/build_sector.smk
@@ -4,8 +4,6 @@
 
 
 rule build_population_layouts:
-    params:
-        logging=config["logging"],
     input:
         nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
         urban_percent="data/urban_percent.csv",
@@ -72,8 +70,6 @@ rule build_simplified_population_layouts:
 if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
 
     rule build_gas_network:
-        params:
-            logging=config["logging"],
         input:
             gas_network="data/gas_network/scigrid-gas/data/IGGIELGN_PipeSegments.geojson",
         output:
@@ -88,8 +84,6 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
             "../scripts/build_gas_network.py"
 
     rule build_gas_input_locations:
-        params:
-            logging=config["logging"],
         input:
             lng=HTTP.remote(
                 "https://globalenergymonitor.org/wp-content/uploads/2022/09/Europe-Gas-Tracker-August-2022.xlsx",
@@ -116,8 +110,6 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
             "../scripts/build_gas_input_locations.py"
 
     rule cluster_gas_network:
-        params:
-            logging=config["logging"],
         input:
             cleaned_gas_network=RESOURCES + "gas_network.csv",
             regions_onshore=RESOURCES
@@ -246,10 +238,8 @@ rule build_solar_thermal_profiles:
 
 rule build_energy_totals:
     params:
-        run=config["run"],
         countries=config["countries"],
         energy=config["energy"],
-        logging=config["logging"],
     input:
         nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
         co2="data/eea/UNFCCC_v23.csv",
@@ -437,10 +427,8 @@ rule build_industry_sector_ratios:
 
 rule build_industrial_production_per_country:
     params:
-        run=config["run"],
         industry=config["industry"],
         countries=config["countries"],
-        logging=config["logging"],
     input:
         ammonia_production=RESOURCES + "ammonia_production.csv",
         jrc="data/jrc-idees-2015",
@@ -491,7 +479,6 @@ rule build_industrial_distribution_key:
     params:
         industry=config["industry"],
         countries=config["countries"],
-        logging=config["logging"],
     input:
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
         clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv",
@@ -567,7 +554,6 @@ rule build_industrial_energy_demand_per_node:
 
 rule build_industrial_energy_demand_per_country_today:
     params:
-        run=config["run"],
         countries=config["countries"],
         industry=config["industry"],
     input:
@@ -723,7 +709,6 @@ rule prepare_sector_network:
         existing_capacities=config["existing_capacities"],
         foresight=config["foresight"],
         costs=config["costs"],
-        logging=config["logging"],
         sector=config["sector"],
         industry=config["industry"],
         pypsa_eur=config["pypsa_eur"],
diff --git a/rules/postprocess.smk b/rules/postprocess.smk
index 1dfdd098..9eb04ef5 100644
--- a/rules/postprocess.smk
+++ b/rules/postprocess.smk
@@ -10,7 +10,6 @@ localrules:
 
 rule plot_network:
     params:
-        logging=config["logging"],
         foresight=config["foresight"],
         plotting=config["plotting"],
     input:
@@ -74,7 +73,6 @@ rule make_summary:
         foresight=config["foresight"],
         costs=config["costs"],
         snapshots=config["snapshots"],
-        logging=config["logging"],
         scenario=config["scenario"],
         RDIR=RDIR,
     input:
@@ -123,7 +121,6 @@ rule make_summary:
 
 rule plot_summary:
     params:
-        logging=config["logging"],
         countries=config["countries"],
         scenario=config["scenario"],
         plotting=config["plotting"],
diff --git a/rules/retrieve.smk b/rules/retrieve.smk
index bc8756d5..4bfbd6c6 100644
--- a/rules/retrieve.smk
+++ b/rules/retrieve.smk
@@ -20,7 +20,6 @@ if config["enable"].get("retrieve_databundle", True):
 
     rule retrieve_databundle:
         params:
-            run=config["run"],
             tutorial=config["tutorial"],
         output:
             expand("data/bundle/{file}", file=datafiles),
diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk
index 2ecba999..49b93a80 100644
--- a/rules/solve_myopic.smk
+++ b/rules/solve_myopic.smk
@@ -7,7 +7,6 @@ rule add_existing_baseyear:
     params:
         scenario=config["scenario"],
         sector=config["sector"],
-        logging=config["logging"],
         existing_capacities=config["existing_capacities"],
         costs=config["costs"],
     input:
@@ -49,7 +48,6 @@ rule add_existing_baseyear:
 
 rule add_brownfield:
     params:
-        logging=config["logging"],
         sector=config["sector"],
         existing_capacities=config["existing_capacities"],
     input:
diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py
index 1d4e3a80..9330953b 100644
--- a/scripts/add_brownfield.py
+++ b/scripts/add_brownfield.py
@@ -139,7 +139,7 @@ if __name__ == "__main__":
             planning_horizons=2030,
         )
 
-    logging.basicConfig(level=snakemake.params["logging"]["level"])
+    logging.basicConfig(level=snakemake.config["logging"]["level"])
 
     update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts)
 
diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py
index a24b078d..a4d3748b 100644
--- a/scripts/add_existing_baseyear.py
+++ b/scripts/add_existing_baseyear.py
@@ -608,7 +608,7 @@ if __name__ == "__main__":
             planning_horizons=2020,
         )
 
-    logging.basicConfig(level=snakemake.params["logging"]["level"])
+    logging.basicConfig(level=snakemake.config["logging"]["level"])
 
     update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts)
 
diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py
index 6cedff97..5099d140 100644
--- a/scripts/build_energy_totals.py
+++ b/scripts/build_energy_totals.py
@@ -373,7 +373,7 @@ def idees_per_country(ct, year, base_dir):
 
 def build_idees(countries, year):
     nprocesses = snakemake.threads
-    disable_progress = snakemake.params["run"].get("disable_progressbar", False)
+    disable_progress = snakemake.config["run"].get("disable_progressbar", False)
 
     func = partial(idees_per_country, year=year, base_dir=snakemake.input.idees)
     tqdm_kwargs = dict(
@@ -735,7 +735,7 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("build_energy_totals")
 
-    logging.basicConfig(level=snakemake.params["logging"]["level"])
+    logging.basicConfig(level=snakemake.config["logging"]["level"])
 
     config = snakemake.params["energy"]
 
diff --git a/scripts/build_gas_input_locations.py b/scripts/build_gas_input_locations.py
index 59f31f5c..a3b945ab 100644
--- a/scripts/build_gas_input_locations.py
+++ b/scripts/build_gas_input_locations.py
@@ -86,7 +86,7 @@ if __name__ == "__main__":
             clusters="37",
         )
 
-    logging.basicConfig(level=snakemake.params["logging"]["level"])
+    logging.basicConfig(level=snakemake.config["logging"]["level"])
 
     regions = load_bus_regions(
         snakemake.input.regions_onshore, snakemake.input.regions_offshore
diff --git a/scripts/build_gas_network.py b/scripts/build_gas_network.py
index a70f15ff..23f58caa 100644
--- a/scripts/build_gas_network.py
+++ b/scripts/build_gas_network.py
@@ -147,7 +147,7 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("build_gas_network")
 
-    logging.basicConfig(level=snakemake.params["logging"]["level"])
+    logging.basicConfig(level=snakemake.config["logging"]["level"])
 
     gas_network = load_dataset(snakemake.input.gas_network)
 
diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py
index 30c652d8..9a513673 100644
--- a/scripts/build_industrial_distribution_key.py
+++ b/scripts/build_industrial_distribution_key.py
@@ -141,7 +141,7 @@ if __name__ == "__main__":
             clusters=48,
         )
 
-    logging.basicConfig(level=snakemake.params["logging"]["level"])
+    logging.basicConfig(level=snakemake.config["logging"]["level"])
 
     countries = snakemake.params["countries"]
 
diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py
index 4fec95cc..c28351c1 100644
--- a/scripts/build_industrial_energy_demand_per_country_today.py
+++ b/scripts/build_industrial_energy_demand_per_country_today.py
@@ -153,7 +153,7 @@ def add_non_eu28_industrial_energy_demand(countries, demand):
 
 def industrial_energy_demand(countries, year):
     nprocesses = snakemake.threads
-    disable_progress = snakemake.params["run"].get("disable_progressbar", False)
+    disable_progress = snakemake.config["run"].get("disable_progressbar", False)
     func = partial(
         industrial_energy_demand_per_country, year=year, jrc_dir=snakemake.input.jrc
     )
diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py
index eb1b16cb..62073ea1 100644
--- a/scripts/build_industrial_production_per_country.py
+++ b/scripts/build_industrial_production_per_country.py
@@ -217,7 +217,7 @@ def industry_production_per_country(country, year, eurostat_dir, jrc_dir):
 
 def industry_production(countries, year, eurostat_dir, jrc_dir):
     nprocesses = snakemake.threads
-    disable_progress = snakemake.params["run"].get("disable_progressbar", False)
+    disable_progress = snakemake.config["run"].get("disable_progressbar", False)
 
     func = partial(
         industry_production_per_country,
@@ -277,7 +277,7 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("build_industrial_production_per_country")
 
-    logging.basicConfig(level=snakemake.params["logging"]["level"])
+    logging.basicConfig(level=snakemake.config["logging"]["level"])
 
     countries = snakemake.params["countries"]
 
diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py
index bc35dbcf..e864d925 100644
--- a/scripts/build_population_layouts.py
+++ b/scripts/build_population_layouts.py
@@ -23,7 +23,7 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("build_population_layouts")
 
-    logging.basicConfig(level=snakemake.params["logging"]["level"])
+    logging.basicConfig(level=snakemake.config["logging"]["level"])
 
     cutout = atlite.Cutout(snakemake.input.cutout)
 
diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py
index 55541522..8fba74e6 100644
--- a/scripts/build_renewable_profiles.py
+++ b/scripts/build_renewable_profiles.py
@@ -203,7 +203,7 @@ if __name__ == "__main__":
     configure_logging(snakemake)
 
     nprocesses = int(snakemake.threads)
-    noprogress = snakemake.params["run"].get("disable_progressbar", True)
+    noprogress = snakemake.config["run"].get("disable_progressbar", True)
     config = snakemake.params["renewable"][snakemake.wildcards.technology]
     resource = config["resource"]  # pv panel config / wind turbine config
     correction_factor = config.get("correction_factor", 1.0)
diff --git a/scripts/cluster_gas_network.py b/scripts/cluster_gas_network.py
index ba11ce1b..e7554dff 100755
--- a/scripts/cluster_gas_network.py
+++ b/scripts/cluster_gas_network.py
@@ -110,7 +110,7 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("cluster_gas_network", simpl="", clusters="37")
 
-    logging.basicConfig(level=snakemake.params["logging"]["level"])
+    logging.basicConfig(level=snakemake.config["logging"]["level"])
 
     fn = snakemake.input.cleaned_gas_network
     df = pd.read_csv(fn, index_col=0)
diff --git a/scripts/make_summary.py b/scripts/make_summary.py
index 56074525..da0712d7 100644
--- a/scripts/make_summary.py
+++ b/scripts/make_summary.py
@@ -682,7 +682,7 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("make_summary")
 
-    logging.basicConfig(level=snakemake.params["logging"]["level"])
+    logging.basicConfig(level=snakemake.config["logging"]["level"])
 
     networks_dict = {
         (cluster, ll, opt + sector_opt, planning_horizon): "results/"
diff --git a/scripts/plot_network.py b/scripts/plot_network.py
index 8aac9db1..399f46e8 100644
--- a/scripts/plot_network.py
+++ b/scripts/plot_network.py
@@ -930,7 +930,7 @@ if __name__ == "__main__":
             planning_horizons="2030",
         )
 
-    logging.basicConfig(level=snakemake.params["logging"]["level"])
+    logging.basicConfig(level=snakemake.config["logging"]["level"])
 
     overrides = override_component_attrs(snakemake.input.overrides)
     n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py
index e18ff61b..4aa37de5 100644
--- a/scripts/plot_summary.py
+++ b/scripts/plot_summary.py
@@ -545,7 +545,7 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("plot_summary")
 
-    logging.basicConfig(level=snakemake.params["logging"]["level"])
+    logging.basicConfig(level=snakemake.config["logging"]["level"])
 
     n_header = 4
 
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index c69f7290..86a9bc82 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -3266,7 +3266,7 @@ if __name__ == "__main__":
             planning_horizons="2030",
         )
 
-    logging.basicConfig(level=snakemake.params["logging"]["level"])
+    logging.basicConfig(level=snakemake.config["logging"]["level"])
 
     update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts)
 
diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py
index 0271166c..de42587d 100644
--- a/scripts/retrieve_databundle.py
+++ b/scripts/retrieve_databundle.py
@@ -63,7 +63,7 @@ if __name__ == "__main__":
     to_fn = Path(f"{rootpath}/data")
 
     logger.info(f"Downloading databundle from '{url}'.")
-    disable_progress = snakemake.params["run"].get("disable_progressbar", False)
+    disable_progress = snakemake.config["run"].get("disable_progressbar", False)
     progress_retrieve(url, tarball_fn, disable=disable_progress)
 
     logger.info("Extracting databundle.")

From 8af1fe5649feb6df991db0d5b991c66fc9e8c25e Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Wed, 17 May 2023 19:25:45 +0200
Subject: [PATCH 105/176] replace the word "config" to "params" in functions

---
 rules/build_electricity.smk                   |  1 +
 scripts/add_electricity.py                    | 47 +++++++++--------
 scripts/build_biomass_potentials.py           |  8 +--
 scripts/build_energy_totals.py                |  6 +--
 scripts/build_hydro_profile.py                |  6 +--
 ...ustrial_energy_demand_per_country_today.py | 10 ++--
 ...build_industrial_production_per_country.py |  8 +--
 ...ustrial_production_per_country_tomorrow.py | 14 +++---
 scripts/build_industry_sector_ratios.py       | 50 +++++++++----------
 scripts/build_renewable_profiles.py           | 48 +++++++++---------
 scripts/prepare_sector_network.py             |  4 +-
 11 files changed, 101 insertions(+), 101 deletions(-)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index 9414c25b..88e7c548 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -277,6 +277,7 @@ rule add_electricity:
         countries=config["countries"],
         renewable=config["renewable"],
         electricity=config["electricity"],
+        conventional=config.get("conventional", {})
         costs=config["costs"],
     input:
         **{
diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index c3e847ce..a3e033e7 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -135,7 +135,7 @@ def _add_missing_carriers_from_costs(n, costs, carriers):
     n.import_components_from_dataframe(emissions, "Carrier")
 
 
-def load_costs(tech_costs, config, elec_config, Nyears=1.0):
+def load_costs(tech_costs, params, elec_params, Nyears=1.0):
     # set all asset costs and other parameters
     costs = pd.read_csv(tech_costs, index_col=[0, 1]).sort_index()
 
@@ -143,7 +143,7 @@ def load_costs(tech_costs, config, elec_config, Nyears=1.0):
     costs.loc[costs.unit.str.contains("/kW"), "value"] *= 1e3
     costs.unit = costs.unit.str.replace("/kW", "/MW")
 
-    fill_values = config["fill_values"]
+    fill_values = params["fill_values"]
     costs = costs.value.unstack().fillna(fill_values)
 
     costs["capital_cost"] = (
@@ -166,8 +166,8 @@ def load_costs(tech_costs, config, elec_config, Nyears=1.0):
     costs.at["CCGT", "co2_emissions"] = costs.at["gas", "co2_emissions"]
 
     costs.at["solar", "capital_cost"] = (
-        config["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"]
-        + (1 - config["rooftop_share"]) * costs.at["solar-utility", "capital_cost"]
+        params["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"]
+        + (1 - params["rooftop_share"]) * costs.at["solar-utility", "capital_cost"]
     )
 
     def costs_for_storage(store, link1, link2=None, max_hours=1.0):
@@ -178,7 +178,7 @@ def load_costs(tech_costs, config, elec_config, Nyears=1.0):
             dict(capital_cost=capital_cost, marginal_cost=0.0, co2_emissions=0.0)
         )
 
-    max_hours = elec_config["max_hours"]
+    max_hours = elec_params["max_hours"]
     costs.loc["battery"] = costs_for_storage(
         costs.loc["battery storage"],
         costs.loc["battery inverter"],
@@ -192,7 +192,7 @@ def load_costs(tech_costs, config, elec_config, Nyears=1.0):
     )
 
     for attr in ("marginal_cost", "capital_cost"):
-        overwrites = config.get(attr)
+        overwrites = params.get(attr)
         if overwrites is not None:
             overwrites = pd.Series(overwrites)
             costs.loc[overwrites.index, attr] = overwrites
@@ -356,7 +356,7 @@ def attach_conventional_generators(
     ppl,
     conventional_carriers,
     extendable_carriers,
-    conventional_config,
+    conventional_params,
     conventional_inputs,
 ):
     carriers = set(conventional_carriers) | set(extendable_carriers["Generator"])
@@ -393,12 +393,12 @@ def attach_conventional_generators(
         lifetime=(ppl.dateout - ppl.datein).fillna(np.inf),
     )
 
-    for carrier in conventional_config:
+    for carrier in conventional_params:
         # Generators with technology affected
         idx = n.generators.query("carrier == @carrier").index
 
-        for attr in list(set(conventional_config[carrier]) & set(n.generators)):
-            values = conventional_config[carrier][attr]
+        for attr in list(set(conventional_params[carrier]) & set(n.generators)):
+            values = conventional_params[carrier][attr]
 
             if f"conventional_{carrier}_{attr}" in conventional_inputs:
                 # Values affecting generators of technology k country-specific
@@ -413,7 +413,7 @@ def attach_conventional_generators(
                 n.generators.loc[idx, attr] = values
 
 
-def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **config):
+def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **params):
     _add_missing_carriers_from_costs(n, costs, carriers)
 
     ppl = (
@@ -468,9 +468,9 @@ def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **con
         )
 
     if "PHS" in carriers and not phs.empty:
-        # fill missing max hours to config value and
+        # fill missing max hours to params value and
         # assume no natural inflow due to lack of data
-        max_hours = config.get("PHS_max_hours", 6)
+        max_hours = params.get("PHS_max_hours", 6)
         phs = phs.replace({"max_hours": {0: max_hours}})
         n.madd(
             "StorageUnit",
@@ -486,7 +486,7 @@ def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **con
         )
 
     if "hydro" in carriers and not hydro.empty:
-        hydro_max_hours = config.get("hydro_max_hours")
+        hydro_max_hours = params.get("hydro_max_hours")
 
         assert hydro_max_hours is not None, "No path for hydro capacities given."
 
@@ -636,13 +636,12 @@ def attach_OPSD_renewables(n, tech_map):
         n.generators.p_nom_min.update(gens.bus.map(caps).dropna())
 
 
-def estimate_renewable_capacities(n, config):
-    year = config["electricity"]["estimate_renewable_capacities"]["year"]
-    tech_map = config["electricity"]["estimate_renewable_capacities"][
+def estimate_renewable_capacities(n, electricity_params, countries):
+    year = electricity_params["estimate_renewable_capacities"]["year"]
+    tech_map = electricity_params["estimate_renewable_capacities"][
         "technology_mapping"
     ]
-    countries = config["countries"]
-    expansion_limit = config["electricity"]["estimate_renewable_capacities"][
+    expansion_limit = electricity_params["estimate_renewable_capacities"][
         "expansion_limit"
     ]
 
@@ -759,7 +758,7 @@ if __name__ == "__main__":
         ppl,
         conventional_carriers,
         extendable_carriers,
-        snakemake.config.get("conventional", {}),
+        snakemake.params.get("conventional", {}),
         conventional_inputs,
     )
 
@@ -773,15 +772,15 @@ if __name__ == "__main__":
     )
 
     if "hydro" in renewable_carriers:
-        conf = snakemake.params["renewable"]["hydro"]
+        para = snakemake.params["renewable"]["hydro"]
         attach_hydro(
             n,
             costs,
             ppl,
             snakemake.input.profile_hydro,
             snakemake.input.hydro_capacities,
-            conf.pop("carriers", []),
-            **conf,
+            para.pop("carriers", []),
+            **para,
         )
 
     if "estimate_renewable_capacities" not in snakemake.params["electricity"]:
@@ -829,7 +828,7 @@ if __name__ == "__main__":
                 "technology_mapping"
             ]
             attach_OPSD_renewables(n, tech_map)
-        estimate_renewable_capacities(n, snakemake.config)
+        estimate_renewable_capacities(n, snakemake.params["electricity"],snakemake.params["countries"])
 
     update_p_nom_max(n)
 
diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py
index 35218e2c..0b423ad5 100644
--- a/scripts/build_biomass_potentials.py
+++ b/scripts/build_biomass_potentials.py
@@ -210,9 +210,9 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("build_biomass_potentials", simpl="", clusters="5")
 
-    config = snakemake.params["biomass"]
-    year = config["year"]
-    scenario = config["scenario"]
+    params = snakemake.params["biomass"]
+    year = params["year"]
+    scenario = params["scenario"]
 
     enspreso = enspreso_biomass_potentials(year, scenario)
 
@@ -228,7 +228,7 @@ if __name__ == "__main__":
 
     df.to_csv(snakemake.output.biomass_potentials_all)
 
-    grouper = {v: k for k, vv in config["classes"].items() for v in vv}
+    grouper = {v: k for k, vv in params["classes"].items() for v in vv}
     df = df.groupby(grouper, axis=1).sum()
 
     df *= 1e6  # TWh/a to MWh/a
diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py
index 5099d140..3e3cb485 100644
--- a/scripts/build_energy_totals.py
+++ b/scripts/build_energy_totals.py
@@ -737,7 +737,7 @@ if __name__ == "__main__":
 
     logging.basicConfig(level=snakemake.config["logging"]["level"])
 
-    config = snakemake.params["energy"]
+    params = snakemake.params["energy"]
 
     nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index("index")
     population = nuts3["pop"].groupby(nuts3.country).sum()
@@ -745,7 +745,7 @@ if __name__ == "__main__":
     countries = snakemake.params["countries"]
     idees_countries = pd.Index(countries).intersection(eu28)
 
-    data_year = config["energy_totals_year"]
+    data_year = params["energy_totals_year"]
     report_year = snakemake.params["energy"]["eurostat_report_year"]
     input_eurostat = snakemake.input.eurostat
     eurostat = build_eurostat(input_eurostat, countries, report_year, data_year)
@@ -755,7 +755,7 @@ if __name__ == "__main__":
     energy = build_energy_totals(countries, eurostat, swiss, idees)
     energy.to_csv(snakemake.output.energy_name)
 
-    base_year_emissions = config["base_emissions_year"]
+    base_year_emissions = params["base_emissions_year"]
     emissions_scope = snakemake.params["energy"]["emissions"]
     eea_co2 = build_eea_co2(snakemake.input.co2, base_year_emissions, emissions_scope)
     eurostat_co2 = build_eurostat_co2(
diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py
index 5453ac5c..dd686be3 100644
--- a/scripts/build_hydro_profile.py
+++ b/scripts/build_hydro_profile.py
@@ -130,7 +130,7 @@ if __name__ == "__main__":
         snakemake = mock_snakemake("build_hydro_profile")
     configure_logging(snakemake)
 
-    config_hydro = snakemake.params["renewable"]["hydro"]
+    params_hydro = snakemake.params["renewable"]["hydro"]
     cutout = atlite.Cutout(snakemake.input.cutout)
 
     countries = snakemake.params["countries"]
@@ -151,7 +151,7 @@ if __name__ == "__main__":
         normalize_using_yearly=eia_stats,
     )
 
-    if "clip_min_inflow" in config_hydro:
-        inflow = inflow.where(inflow > config_hydro["clip_min_inflow"], 0)
+    if "clip_min_inflow" in params_hydro:
+        inflow = inflow.where(inflow > params_hydro["clip_min_inflow"], 0)
 
     inflow.to_netcdf(snakemake.output[0])
diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py
index c28351c1..9f8c47d0 100644
--- a/scripts/build_industrial_energy_demand_per_country_today.py
+++ b/scripts/build_industrial_energy_demand_per_country_today.py
@@ -101,8 +101,8 @@ def add_ammonia_energy_demand(demand):
 
     def get_ammonia_by_fuel(x):
         fuels = {
-            "gas": config["MWh_CH4_per_tNH3_SMR"],
-            "electricity": config["MWh_elec_per_tNH3_SMR"],
+            "gas": params["MWh_CH4_per_tNH3_SMR"],
+            "electricity": params["MWh_elec_per_tNH3_SMR"],
         }
 
         return pd.Series({k: x * v for k, v in fuels.items()})
@@ -112,7 +112,7 @@ def add_ammonia_energy_demand(demand):
         index=demand.index, fill_value=0.0
     )
 
-    ammonia = pd.DataFrame({"ammonia": ammonia * config["MWh_NH3_per_tNH3"]}).T
+    ammonia = pd.DataFrame({"ammonia": ammonia * params["MWh_NH3_per_tNH3"]}).T
 
     demand["Ammonia"] = ammonia.unstack().reindex(index=demand.index, fill_value=0.0)
 
@@ -178,8 +178,8 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("build_industrial_energy_demand_per_country_today")
 
-    config = snakemake.params["industry"]
-    year = config.get("reference_year", 2015)
+    params = snakemake.params["industry"]
+    year = params.get("reference_year", 2015)
     countries = pd.Index(snakemake.params["countries"])
 
     demand = industrial_energy_demand(countries.intersection(eu28), year)
diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py
index 62073ea1..889c9ecd 100644
--- a/scripts/build_industrial_production_per_country.py
+++ b/scripts/build_industrial_production_per_country.py
@@ -264,9 +264,9 @@ def separate_basic_chemicals(demand, year):
 
     # assume HVC, methanol, chlorine production proportional to non-ammonia basic chemicals
     distribution_key = demand["Basic chemicals"] / demand["Basic chemicals"].sum()
-    demand["HVC"] = config["HVC_production_today"] * 1e3 * distribution_key
-    demand["Chlorine"] = config["chlorine_production_today"] * 1e3 * distribution_key
-    demand["Methanol"] = config["methanol_production_today"] * 1e3 * distribution_key
+    demand["HVC"] = params["HVC_production_today"] * 1e3 * distribution_key
+    demand["Chlorine"] = params["chlorine_production_today"] * 1e3 * distribution_key
+    demand["Methanol"] = params["methanol_production_today"] * 1e3 * distribution_key
 
     demand.drop(columns=["Basic chemicals"], inplace=True)
 
@@ -283,7 +283,7 @@ if __name__ == "__main__":
 
     year = snakemake.params["industry"]["reference_year"]
 
-    config = snakemake.params["industry"]
+    params = snakemake.params["industry"]
 
     jrc_dir = snakemake.input.jrc
     eurostat_dir = snakemake.input.eurostat
diff --git a/scripts/build_industrial_production_per_country_tomorrow.py b/scripts/build_industrial_production_per_country_tomorrow.py
index b9ac9b16..609170aa 100644
--- a/scripts/build_industrial_production_per_country_tomorrow.py
+++ b/scripts/build_industrial_production_per_country_tomorrow.py
@@ -15,7 +15,7 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("build_industrial_production_per_country_tomorrow")
 
-    config = snakemake.params["industry"]
+    params = snakemake.params["industry"]
 
     investment_year = int(snakemake.wildcards.planning_horizons)
 
@@ -25,8 +25,8 @@ if __name__ == "__main__":
     keys = ["Integrated steelworks", "Electric arc"]
     total_steel = production[keys].sum(axis=1)
 
-    st_primary_fraction = get(config["St_primary_fraction"], investment_year)
-    dri_fraction = get(config["DRI_fraction"], investment_year)
+    st_primary_fraction = get(params["St_primary_fraction"], investment_year)
+    dri_fraction = get(params["DRI_fraction"], investment_year)
     int_steel = production["Integrated steelworks"].sum()
     fraction_persistent_primary = st_primary_fraction * total_steel.sum() / int_steel
 
@@ -51,7 +51,7 @@ if __name__ == "__main__":
     key_pri = "Aluminium - primary production"
     key_sec = "Aluminium - secondary production"
 
-    al_primary_fraction = get(config["Al_primary_fraction"], investment_year)
+    al_primary_fraction = get(params["Al_primary_fraction"], investment_year)
     fraction_persistent_primary = (
         al_primary_fraction * total_aluminium.sum() / production[key_pri].sum()
     )
@@ -60,15 +60,15 @@ if __name__ == "__main__":
     production[key_sec] = total_aluminium - production[key_pri]
 
     production["HVC (mechanical recycling)"] = (
-        get(config["HVC_mechanical_recycling_fraction"], investment_year)
+        get(params["HVC_mechanical_recycling_fraction"], investment_year)
         * production["HVC"]
     )
     production["HVC (chemical recycling)"] = (
-        get(config["HVC_chemical_recycling_fraction"], investment_year)
+        get(params["HVC_chemical_recycling_fraction"], investment_year)
         * production["HVC"]
     )
 
-    production["HVC"] *= get(config["HVC_primary_fraction"], investment_year)
+    production["HVC"] *= get(params["HVC_primary_fraction"], investment_year)
 
     fn = snakemake.output.industrial_production_per_country_tomorrow
     production.to_csv(fn, float_format="%.2f")
diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py
index c3e2bd49..2ec007a9 100644
--- a/scripts/build_industry_sector_ratios.py
+++ b/scripts/build_industry_sector_ratios.py
@@ -185,10 +185,10 @@ def iron_and_steel():
     df[sector] = df["Electric arc"]
 
     # add H2 consumption for DRI at 1.7 MWh H2 /ton steel
-    df.at["hydrogen", sector] = config["H2_DRI"]
+    df.at["hydrogen", sector] = params["H2_DRI"]
 
     # add electricity consumption in DRI shaft (0.322 MWh/tSl)
-    df.at["elec", sector] += config["elec_DRI"]
+    df.at["elec", sector] += params["elec_DRI"]
 
     ## Integrated steelworks
     # could be used in combination with CCS)
@@ -383,19 +383,19 @@ def chemicals_industry():
     assert s_emi.index[0] == sector
 
     # convert from MtHVC/a to ktHVC/a
-    s_out = config["HVC_production_today"] * 1e3
+    s_out = params["HVC_production_today"] * 1e3
 
     # tCO2/t material
     df.loc["process emission", sector] += (
         s_emi["Process emissions"]
-        - config["petrochemical_process_emissions"] * 1e3
-        - config["NH3_process_emissions"] * 1e3
+        - params["petrochemical_process_emissions"] * 1e3
+        - params["NH3_process_emissions"] * 1e3
     ) / s_out
 
     # emissions originating from feedstock, could be non-fossil origin
     # tCO2/t material
     df.loc["process emission from feedstock", sector] += (
-        config["petrochemical_process_emissions"] * 1e3
+        params["petrochemical_process_emissions"] * 1e3
     ) / s_out
 
     # convert from ktoe/a to GWh/a
@@ -405,18 +405,18 @@ def chemicals_industry():
     # subtract ammonia energy demand (in ktNH3/a)
     ammonia = pd.read_csv(snakemake.input.ammonia_production, index_col=0)
     ammonia_total = ammonia.loc[ammonia.index.intersection(eu28), str(year)].sum()
-    df.loc["methane", sector] -= ammonia_total * config["MWh_CH4_per_tNH3_SMR"]
-    df.loc["elec", sector] -= ammonia_total * config["MWh_elec_per_tNH3_SMR"]
+    df.loc["methane", sector] -= ammonia_total * params["MWh_CH4_per_tNH3_SMR"]
+    df.loc["elec", sector] -= ammonia_total * params["MWh_elec_per_tNH3_SMR"]
 
     # subtract chlorine demand
-    chlorine_total = config["chlorine_production_today"]
-    df.loc["hydrogen", sector] -= chlorine_total * config["MWh_H2_per_tCl"]
-    df.loc["elec", sector] -= chlorine_total * config["MWh_elec_per_tCl"]
+    chlorine_total = params["chlorine_production_today"]
+    df.loc["hydrogen", sector] -= chlorine_total * params["MWh_H2_per_tCl"]
+    df.loc["elec", sector] -= chlorine_total * params["MWh_elec_per_tCl"]
 
     # subtract methanol demand
-    methanol_total = config["methanol_production_today"]
-    df.loc["methane", sector] -= methanol_total * config["MWh_CH4_per_tMeOH"]
-    df.loc["elec", sector] -= methanol_total * config["MWh_elec_per_tMeOH"]
+    methanol_total = params["methanol_production_today"]
+    df.loc["methane", sector] -= methanol_total * params["MWh_CH4_per_tMeOH"]
+    df.loc["elec", sector] -= methanol_total * params["MWh_elec_per_tMeOH"]
 
     # MWh/t material
     df.loc[sources, sector] = df.loc[sources, sector] / s_out
@@ -427,37 +427,37 @@ def chemicals_industry():
 
     sector = "HVC (mechanical recycling)"
     df[sector] = 0.0
-    df.loc["elec", sector] = config["MWh_elec_per_tHVC_mechanical_recycling"]
+    df.loc["elec", sector] = params["MWh_elec_per_tHVC_mechanical_recycling"]
 
     # HVC chemical recycling
 
     sector = "HVC (chemical recycling)"
     df[sector] = 0.0
-    df.loc["elec", sector] = config["MWh_elec_per_tHVC_chemical_recycling"]
+    df.loc["elec", sector] = params["MWh_elec_per_tHVC_chemical_recycling"]
 
     # Ammonia
 
     sector = "Ammonia"
     df[sector] = 0.0
     if snakemake.params["sector"].get("ammonia", False):
-        df.loc["ammonia", sector] = config["MWh_NH3_per_tNH3"]
+        df.loc["ammonia", sector] = params["MWh_NH3_per_tNH3"]
     else:
-        df.loc["hydrogen", sector] = config["MWh_H2_per_tNH3_electrolysis"]
-        df.loc["elec", sector] = config["MWh_elec_per_tNH3_electrolysis"]
+        df.loc["hydrogen", sector] = params["MWh_H2_per_tNH3_electrolysis"]
+        df.loc["elec", sector] = params["MWh_elec_per_tNH3_electrolysis"]
 
     # Chlorine
 
     sector = "Chlorine"
     df[sector] = 0.0
-    df.loc["hydrogen", sector] = config["MWh_H2_per_tCl"]
-    df.loc["elec", sector] = config["MWh_elec_per_tCl"]
+    df.loc["hydrogen", sector] = params["MWh_H2_per_tCl"]
+    df.loc["elec", sector] = params["MWh_elec_per_tCl"]
 
     # Methanol
 
     sector = "Methanol"
     df[sector] = 0.0
-    df.loc["methane", sector] = config["MWh_CH4_per_tMeOH"]
-    df.loc["elec", sector] = config["MWh_elec_per_tMeOH"]
+    df.loc["methane", sector] = params["MWh_CH4_per_tMeOH"]
+    df.loc["elec", sector] = params["MWh_elec_per_tMeOH"]
 
     # Other chemicals
 
@@ -1465,10 +1465,10 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("build_industry_sector_ratios")
 
-    # TODO make config option
+    # TODO make params option
     year = 2015
 
-    config = snakemake.params["industry"]
+    params = snakemake.params["industry"]
 
     df = pd.concat(
         [
diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py
index 8fba74e6..91463c23 100644
--- a/scripts/build_renewable_profiles.py
+++ b/scripts/build_renewable_profiles.py
@@ -64,7 +64,7 @@ Inputs
 - ``resources/offshore_shapes.geojson``: confer :ref:`shapes`
 - ``resources/regions_onshore.geojson``: (if not offshore wind), confer :ref:`busregions`
 - ``resources/regions_offshore.geojson``: (if offshore wind), :ref:`busregions`
-- ``"cutouts/" + config["renewable"][{technology}]['cutout']``: :ref:`cutout`
+- ``"cutouts/" + params["renewable"][{technology}]['cutout']``: :ref:`cutout`
 - ``networks/base.nc``: :ref:`base`
 
 Outputs
@@ -204,14 +204,14 @@ if __name__ == "__main__":
 
     nprocesses = int(snakemake.threads)
     noprogress = snakemake.config["run"].get("disable_progressbar", True)
-    config = snakemake.params["renewable"][snakemake.wildcards.technology]
-    resource = config["resource"]  # pv panel config / wind turbine config
-    correction_factor = config.get("correction_factor", 1.0)
-    capacity_per_sqkm = config["capacity_per_sqkm"]
-    p_nom_max_meth = config.get("potential", "conservative")
+    params = snakemake.params["renewable"][snakemake.wildcards.technology]
+    resource = params["resource"]  # pv panel params / wind turbine params
+    correction_factor = params.get("correction_factor", 1.0)
+    capacity_per_sqkm = params["capacity_per_sqkm"]
+    p_nom_max_meth = params.get("potential", "conservative")
 
-    if isinstance(config.get("corine", {}), list):
-        config["corine"] = {"grid_codes": config["corine"]}
+    if isinstance(params.get("corine", {}), list):
+        params["corine"] = {"grid_codes": params["corine"]}
 
     if correction_factor != 1.0:
         logger.info(f"correction_factor is set as {correction_factor}")
@@ -229,13 +229,13 @@ if __name__ == "__main__":
     regions = regions.set_index("name").rename_axis("bus")
     buses = regions.index
 
-    res = config.get("excluder_resolution", 100)
+    res = params.get("excluder_resolution", 100)
     excluder = atlite.ExclusionContainer(crs=3035, res=res)
 
-    if config["natura"]:
+    if params["natura"]:
         excluder.add_raster(snakemake.input.natura, nodata=0, allow_no_overlap=True)
 
-    corine = config.get("corine", {})
+    corine = params.get("corine", {})
     if "grid_codes" in corine:
         codes = corine["grid_codes"]
         excluder.add_raster(snakemake.input.corine, codes=codes, invert=True, crs=3035)
@@ -246,28 +246,28 @@ if __name__ == "__main__":
             snakemake.input.corine, codes=codes, buffer=buffer, crs=3035
         )
 
-    if "ship_threshold" in config:
+    if "ship_threshold" in params:
         shipping_threshold = (
-            config["ship_threshold"] * 8760 * 6
+            params["ship_threshold"] * 8760 * 6
         )  # approximation because 6 years of data which is hourly collected
         func = functools.partial(np.less, shipping_threshold)
         excluder.add_raster(
             snakemake.input.ship_density, codes=func, crs=4326, allow_no_overlap=True
         )
 
-    if config.get("max_depth"):
+    if params.get("max_depth"):
         # lambda not supported for atlite + multiprocessing
         # use named function np.greater with partially frozen argument instead
         # and exclude areas where: -max_depth > grid cell depth
-        func = functools.partial(np.greater, -config["max_depth"])
+        func = functools.partial(np.greater, -params["max_depth"])
         excluder.add_raster(snakemake.input.gebco, codes=func, crs=4326, nodata=-1000)
 
-    if "min_shore_distance" in config:
-        buffer = config["min_shore_distance"]
+    if "min_shore_distance" in params:
+        buffer = params["min_shore_distance"]
         excluder.add_geometry(snakemake.input.country_shapes, buffer=buffer)
 
-    if "max_shore_distance" in config:
-        buffer = config["max_shore_distance"]
+    if "max_shore_distance" in params:
+        buffer = params["max_shore_distance"]
         excluder.add_geometry(
             snakemake.input.country_shapes, buffer=buffer, invert=True
         )
@@ -309,7 +309,7 @@ if __name__ == "__main__":
         p_nom_max = capacities / max_cap_factor
     else:
         raise AssertionError(
-            'Config key `potential` should be one of "simple" '
+            'params key `potential` should be one of "simple" '
             f'(default) or "conservative", not "{p_nom_max_meth}"'
         )
 
@@ -358,13 +358,13 @@ if __name__ == "__main__":
     # select only buses with some capacity and minimal capacity factor
     ds = ds.sel(
         bus=(
-            (ds["profile"].mean("time") > config.get("min_p_max_pu", 0.0))
-            & (ds["p_nom_max"] > config.get("min_p_nom_max", 0.0))
+            (ds["profile"].mean("time") > params.get("min_p_max_pu", 0.0))
+            & (ds["p_nom_max"] > params.get("min_p_nom_max", 0.0))
         )
     )
 
-    if "clip_p_max_pu" in config:
-        min_p_max_pu = config["clip_p_max_pu"]
+    if "clip_p_max_pu" in params:
+        min_p_max_pu = params["clip_p_max_pu"]
         ds["profile"] = ds["profile"].where(ds["profile"] >= min_p_max_pu, 0)
 
     ds.to_netcdf(snakemake.output.profile)
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 86a9bc82..1db9b916 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -727,7 +727,7 @@ def cycling_shift(df, steps=1):
     return df
 
 
-def prepare_costs(cost_file, config, nyears):
+def prepare_costs(cost_file, params, nyears):
     # set all asset costs and other parameters
     costs = pd.read_csv(cost_file, index_col=[0, 1]).sort_index()
 
@@ -739,7 +739,7 @@ def prepare_costs(cost_file, config, nyears):
         costs.loc[:, "value"].unstack(level=1).groupby("technology").sum(min_count=1)
     )
 
-    costs = costs.fillna(config["fill_values"])
+    costs = costs.fillna(params["fill_values"])
 
     def annuity_factor(v):
         return annuity(v["lifetime"], v["discount rate"]) + v["FOM"] / 100

From 6366dc4c55076c084caddcbd67f997c17e9360bd Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 17 May 2023 17:26:00 +0000
Subject: [PATCH 106/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/add_electricity.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index a3e033e7..adb8bf31 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -638,9 +638,7 @@ def attach_OPSD_renewables(n, tech_map):
 
 def estimate_renewable_capacities(n, electricity_params, countries):
     year = electricity_params["estimate_renewable_capacities"]["year"]
-    tech_map = electricity_params["estimate_renewable_capacities"][
-        "technology_mapping"
-    ]
+    tech_map = electricity_params["estimate_renewable_capacities"]["technology_mapping"]
     expansion_limit = electricity_params["estimate_renewable_capacities"][
         "expansion_limit"
     ]
@@ -828,7 +826,9 @@ if __name__ == "__main__":
                 "technology_mapping"
             ]
             attach_OPSD_renewables(n, tech_map)
-        estimate_renewable_capacities(n, snakemake.params["electricity"],snakemake.params["countries"])
+        estimate_renewable_capacities(
+            n, snakemake.params["electricity"], snakemake.params["countries"]
+        )
 
     update_p_nom_max(n)
 

From d97a5434630d93c2f41ab64c8304ccc94961bee9 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Wed, 17 May 2023 19:37:47 +0200
Subject: [PATCH 107/176] missing a coma in build_electricity.smk

---
 rules/build_electricity.smk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index 88e7c548..1ba1a1c5 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -277,7 +277,7 @@ rule add_electricity:
         countries=config["countries"],
         renewable=config["renewable"],
         electricity=config["electricity"],
-        conventional=config.get("conventional", {})
+        conventional=config.get("conventional", {}),
         costs=config["costs"],
     input:
         **{

From c043100ada790d088ea190e9f8ba634e28a204e1 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Fri, 19 May 2023 15:42:07 +0200
Subject: [PATCH 108/176] add more params in simplify_network.py

---
 rules/build_electricity.smk |  8 +++-
 scripts/add_electricity.py  |  2 +-
 scripts/cluster_network.py  |  2 +-
 scripts/simplify_network.py | 76 +++++++++++++++++++------------------
 4 files changed, 49 insertions(+), 39 deletions(-)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index 1ba1a1c5..f33d1686 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -318,6 +318,12 @@ rule simplify_network:
         clustering=config["clustering"],
         electricity=config["electricity"],
         costs=config["costs"],
+        renewable=config["renewable"],
+        length_factor=config["lines"]["length_factor"],
+        p_max_pu=config["links"].get("p_max_pu", 1.0),
+        exclude_carriers=config["clustering"]["simplify_network"].get("exclude_carriers", []),
+        focus_weights=config.get("focus_weights", None),
+        solver_name=config["solving"]["solver"]["name"],
     input:
         network=RESOURCES + "networks/elec.nc",
         tech_costs=COSTS,
@@ -350,7 +356,7 @@ rule cluster_network:
         lines=config["lines"],
         renewable=config["renewable"],
         clustering=config["clustering"],
-        enable=config["enable"].get("custom_busmap", False),
+        custom_busmap=config["enable"].get("custom_busmap", False),
     input:
         network=RESOURCES + "networks/elec_s{simpl}.nc",
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson",
diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index adb8bf31..3717f0e8 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -756,7 +756,7 @@ if __name__ == "__main__":
         ppl,
         conventional_carriers,
         extendable_carriers,
-        snakemake.params.get("conventional", {}),
+        snakemake.params["conventional"],
         conventional_inputs,
     )
 
diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py
index 78798804..d217e5ed 100644
--- a/scripts/cluster_network.py
+++ b/scripts/cluster_network.py
@@ -521,7 +521,7 @@ if __name__ == "__main__":
             for p in aggregation_strategies.keys()
         }
 
-        custom_busmap = snakemake.params["enable"].get("custom_busmap", False)
+        custom_busmap = snakemake.params["custom_busmap"]
         if custom_busmap:
             custom_busmap = pd.read_csv(
                 snakemake.input.custom_busmap, index_col=0, squeeze=True
diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index bb7f615c..91dde6b1 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -149,17 +149,17 @@ def simplify_network_to_380(n):
     return n, trafo_map
 
 
-def _prepare_connection_costs_per_link(n, costs, config):
+def _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param):
     if n.links.empty:
         return {}
 
     connection_costs_per_link = {}
 
-    for tech in config["renewable"]:
+    for tech in renewable_param:
         if tech.startswith("offwind"):
             connection_costs_per_link[tech] = (
                 n.links.length
-                * config["lines"]["length_factor"]
+                * length_factor_param
                 * (
                     n.links.underwater_fraction
                     * costs.at[tech + "-connection-submarine", "capital_cost"]
@@ -172,10 +172,10 @@ def _prepare_connection_costs_per_link(n, costs, config):
 
 
 def _compute_connection_costs_to_bus(
-    n, busmap, costs, config, connection_costs_per_link=None, buses=None
+    n, busmap, costs, renewable_param, length_factor_param, connection_costs_per_link=None, buses=None
 ):
     if connection_costs_per_link is None:
-        connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, config)
+        connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param)
 
     if buses is None:
         buses = busmap.index[busmap.index != busmap.values]
@@ -265,7 +265,7 @@ def _aggregate_and_move_components(
         n.mremove(c, df.index[df.bus0.isin(buses_to_del) | df.bus1.isin(buses_to_del)])
 
 
-def simplify_links(n, costs, config, output, aggregation_strategies=dict()):
+def simplify_links(n, costs, renewable_param, length_factor_param, p_max_pu_param, exclude_carriers_param, output, aggregation_strategies=dict()):
     ## Complex multi-node links are folded into end-points
     logger.info("Simplifying connected link components")
 
@@ -315,7 +315,7 @@ def simplify_links(n, costs, config, output, aggregation_strategies=dict()):
 
     busmap = n.buses.index.to_series()
 
-    connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, config)
+    connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param)
     connection_costs_to_bus = pd.DataFrame(
         0.0, index=n.buses.index, columns=list(connection_costs_per_link)
     )
@@ -333,12 +333,11 @@ def simplify_links(n, costs, config, output, aggregation_strategies=dict()):
             )
             busmap.loc[buses] = b[np.r_[0, m.argmin(axis=0), 1]]
             connection_costs_to_bus.loc[buses] += _compute_connection_costs_to_bus(
-                n, busmap, costs, config, connection_costs_per_link, buses
+                n, busmap, costs, renewable_param, length_factor_param, connection_costs_per_link, buses
             )
 
             all_links = [i for _, i in sum(links, [])]
 
-            p_max_pu = config["links"].get("p_max_pu", 1.0)
             lengths = n.links.loc[all_links, "length"]
             name = lengths.idxmax() + "+{}".format(len(links) - 1)
             params = dict(
@@ -354,8 +353,8 @@ def simplify_links(n, costs, config, output, aggregation_strategies=dict()):
                     / lengths.sum()
                     * n.links.loc[all_links, "underwater_fraction"]
                 ),
-                p_max_pu=p_max_pu,
-                p_min_pu=-p_max_pu,
+                p_max_pu=p_max_pu_param,
+                p_min_pu=-p_max_pu_param,
                 underground=False,
                 under_construction=False,
             )
@@ -377,33 +376,29 @@ def simplify_links(n, costs, config, output, aggregation_strategies=dict()):
 
     logger.debug("Collecting all components using the busmap")
 
-    exclude_carriers = config["clustering"]["simplify_network"].get(
-        "exclude_carriers", []
-    )
-
     _aggregate_and_move_components(
         n,
         busmap,
         connection_costs_to_bus,
         output,
         aggregation_strategies=aggregation_strategies,
-        exclude_carriers=exclude_carriers,
+        exclude_carriers=exclude_carriers_param,
     )
     return n, busmap
 
 
-def remove_stubs(n, costs, config, output, aggregation_strategies=dict()):
+def remove_stubs(n, costs, renewable_param, length_factor_param, clustering_param, exclude_carriers_param, output, aggregation_strategies=dict()):
     logger.info("Removing stubs")
 
-    across_borders = config["clustering"]["simplify_network"].get(
+    across_borders = clustering_param["simplify_network"].get(
         "remove_stubs_across_borders", True
     )
     matching_attrs = [] if across_borders else ["country"]
     busmap = busmap_by_stubs(n, matching_attrs)
 
-    connection_costs_to_bus = _compute_connection_costs_to_bus(n, busmap, costs, config)
+    connection_costs_to_bus = _compute_connection_costs_to_bus(n, busmap, costs, renewable_param, length_factor_param)
 
-    exclude_carriers = config["clustering"]["simplify_network"].get(
+    exclude_carriers = clustering_param["simplify_network"].get(
         "exclude_carriers", []
     )
 
@@ -473,17 +468,15 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None):
 
 
 def cluster(
-    n, n_clusters, config, algorithm="hac", feature=None, aggregation_strategies=dict()
+    n, n_clusters, focus_weights_param, renewable_param, solver_name_param, algorithm="hac", feature=None, aggregation_strategies=dict()
 ):
     logger.info(f"Clustering to {n_clusters} buses")
 
-    focus_weights = config.get("focus_weights", None)
-
     renewable_carriers = pd.Index(
         [
             tech
             for tech in n.generators.carrier.unique()
-            if tech.split("-", 2)[0] in config["renewable"]
+            if tech.split("-", 2)[0] in renewable_param
         ]
     )
 
@@ -492,10 +485,10 @@ def cluster(
         n_clusters,
         custom_busmap=False,
         aggregation_strategies=aggregation_strategies,
-        solver_name=config["solving"]["solver"]["name"],
+        solver_name=solver_name_param,
         algorithm=algorithm,
         feature=feature,
-        focus_weights=focus_weights,
+        focus_weights=focus_weights_param,
     )
 
     return clustering.network, clustering.busmap
@@ -531,23 +524,32 @@ if __name__ == "__main__":
     )
 
     n, simplify_links_map = simplify_links(
-        n, technology_costs, snakemake.config, snakemake.output, aggregation_strategies
+        n, technology_costs, 
+        snakemake.params['renewable'], 
+        snakemake.params['length_factor'],
+        snakemake.params['p_max_pu'],
+        snakemake.params['exclude_carriers'],
+        snakemake.output, 
+        aggregation_strategies
     )
 
     busmaps = [trafo_map, simplify_links_map]
 
-    cluster_config = snakemake.params["clustering"]["simplify_network"]
-    if cluster_config.get("remove_stubs", True):
+    cluster_param = snakemake.params["clustering"]["simplify_network"]
+    if cluster_param.get("remove_stubs", True):
         n, stub_map = remove_stubs(
             n,
             technology_costs,
-            snakemake.config,
+            snakemake.params['renewable'], 
+            snakemake.params['length_factor'],
+            snakemake.params["clustering"],
+            snakemake.params['exclude_carriers'],
             snakemake.output,
             aggregation_strategies=aggregation_strategies,
         )
         busmaps.append(stub_map)
 
-    if cluster_config.get("to_substations", False):
+    if cluster_param.get("to_substations", False):
         n, substation_map = aggregate_to_substations(n, aggregation_strategies)
         busmaps.append(substation_map)
 
@@ -558,10 +560,10 @@ if __name__ == "__main__":
         .get("cluster_network", {})
         .get("algorithm", "hac")
         == "hac"
-        or cluster_config.get("algorithm", "hac") == "hac"
+        or cluster_param.get("algorithm", "hac") == "hac"
     ):
         carriers = (
-            cluster_config.get("feature", "solar+onwind-time").split("-")[0].split("+")
+            cluster_param.get("feature", "solar+onwind-time").split("-")[0].split("+")
         )
         for carrier in carriers:
             buses_i = list(
@@ -577,9 +579,11 @@ if __name__ == "__main__":
         n, cluster_map = cluster(
             n,
             int(snakemake.wildcards.simpl),
-            snakemake.config,
-            cluster_config.get("algorithm", "hac"),
-            cluster_config.get("feature", None),
+            snakemake.params['focus_weights'],
+            snakemake.params['renewable'],
+            snakemake.params['solver_name'],
+            cluster_param.get("algorithm", "hac"),
+            cluster_param.get("feature", None),
             aggregation_strategies,
         )
         busmaps.append(cluster_map)

From 216a02fba12fe445e36fae1c019820715379c437 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 19 May 2023 13:42:43 +0000
Subject: [PATCH 109/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 rules/build_electricity.smk |  4 +-
 scripts/simplify_network.py | 90 +++++++++++++++++++++++++++----------
 2 files changed, 69 insertions(+), 25 deletions(-)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index f33d1686..45436c5d 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -321,7 +321,9 @@ rule simplify_network:
         renewable=config["renewable"],
         length_factor=config["lines"]["length_factor"],
         p_max_pu=config["links"].get("p_max_pu", 1.0),
-        exclude_carriers=config["clustering"]["simplify_network"].get("exclude_carriers", []),
+        exclude_carriers=config["clustering"]["simplify_network"].get(
+            "exclude_carriers", []
+        ),
         focus_weights=config.get("focus_weights", None),
         solver_name=config["solving"]["solver"]["name"],
     input:
diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 91dde6b1..689e8084 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -172,10 +172,18 @@ def _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_
 
 
 def _compute_connection_costs_to_bus(
-    n, busmap, costs, renewable_param, length_factor_param, connection_costs_per_link=None, buses=None
+    n,
+    busmap,
+    costs,
+    renewable_param,
+    length_factor_param,
+    connection_costs_per_link=None,
+    buses=None,
 ):
     if connection_costs_per_link is None:
-        connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param)
+        connection_costs_per_link = _prepare_connection_costs_per_link(
+            n, costs, renewable_param, length_factor_param
+        )
 
     if buses is None:
         buses = busmap.index[busmap.index != busmap.values]
@@ -265,7 +273,16 @@ def _aggregate_and_move_components(
         n.mremove(c, df.index[df.bus0.isin(buses_to_del) | df.bus1.isin(buses_to_del)])
 
 
-def simplify_links(n, costs, renewable_param, length_factor_param, p_max_pu_param, exclude_carriers_param, output, aggregation_strategies=dict()):
+def simplify_links(
+    n,
+    costs,
+    renewable_param,
+    length_factor_param,
+    p_max_pu_param,
+    exclude_carriers_param,
+    output,
+    aggregation_strategies=dict(),
+):
     ## Complex multi-node links are folded into end-points
     logger.info("Simplifying connected link components")
 
@@ -315,7 +332,9 @@ def simplify_links(n, costs, renewable_param, length_factor_param, p_max_pu_para
 
     busmap = n.buses.index.to_series()
 
-    connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param)
+    connection_costs_per_link = _prepare_connection_costs_per_link(
+        n, costs, renewable_param, length_factor_param
+    )
     connection_costs_to_bus = pd.DataFrame(
         0.0, index=n.buses.index, columns=list(connection_costs_per_link)
     )
@@ -333,7 +352,13 @@ def simplify_links(n, costs, renewable_param, length_factor_param, p_max_pu_para
             )
             busmap.loc[buses] = b[np.r_[0, m.argmin(axis=0), 1]]
             connection_costs_to_bus.loc[buses] += _compute_connection_costs_to_bus(
-                n, busmap, costs, renewable_param, length_factor_param, connection_costs_per_link, buses
+                n,
+                busmap,
+                costs,
+                renewable_param,
+                length_factor_param,
+                connection_costs_per_link,
+                buses,
             )
 
             all_links = [i for _, i in sum(links, [])]
@@ -387,7 +412,16 @@ def simplify_links(n, costs, renewable_param, length_factor_param, p_max_pu_para
     return n, busmap
 
 
-def remove_stubs(n, costs, renewable_param, length_factor_param, clustering_param, exclude_carriers_param, output, aggregation_strategies=dict()):
+def remove_stubs(
+    n,
+    costs,
+    renewable_param,
+    length_factor_param,
+    clustering_param,
+    exclude_carriers_param,
+    output,
+    aggregation_strategies=dict(),
+):
     logger.info("Removing stubs")
 
     across_borders = clustering_param["simplify_network"].get(
@@ -396,12 +430,12 @@ def remove_stubs(n, costs, renewable_param, length_factor_param, clustering_para
     matching_attrs = [] if across_borders else ["country"]
     busmap = busmap_by_stubs(n, matching_attrs)
 
-    connection_costs_to_bus = _compute_connection_costs_to_bus(n, busmap, costs, renewable_param, length_factor_param)
-
-    exclude_carriers = clustering_param["simplify_network"].get(
-        "exclude_carriers", []
+    connection_costs_to_bus = _compute_connection_costs_to_bus(
+        n, busmap, costs, renewable_param, length_factor_param
     )
 
+    exclude_carriers = clustering_param["simplify_network"].get("exclude_carriers", [])
+
     _aggregate_and_move_components(
         n,
         busmap,
@@ -468,7 +502,14 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None):
 
 
 def cluster(
-    n, n_clusters, focus_weights_param, renewable_param, solver_name_param, algorithm="hac", feature=None, aggregation_strategies=dict()
+    n,
+    n_clusters,
+    focus_weights_param,
+    renewable_param,
+    solver_name_param,
+    algorithm="hac",
+    feature=None,
+    aggregation_strategies=dict(),
 ):
     logger.info(f"Clustering to {n_clusters} buses")
 
@@ -524,13 +565,14 @@ if __name__ == "__main__":
     )
 
     n, simplify_links_map = simplify_links(
-        n, technology_costs, 
-        snakemake.params['renewable'], 
-        snakemake.params['length_factor'],
-        snakemake.params['p_max_pu'],
-        snakemake.params['exclude_carriers'],
-        snakemake.output, 
-        aggregation_strategies
+        n,
+        technology_costs,
+        snakemake.params["renewable"],
+        snakemake.params["length_factor"],
+        snakemake.params["p_max_pu"],
+        snakemake.params["exclude_carriers"],
+        snakemake.output,
+        aggregation_strategies,
     )
 
     busmaps = [trafo_map, simplify_links_map]
@@ -540,10 +582,10 @@ if __name__ == "__main__":
         n, stub_map = remove_stubs(
             n,
             technology_costs,
-            snakemake.params['renewable'], 
-            snakemake.params['length_factor'],
+            snakemake.params["renewable"],
+            snakemake.params["length_factor"],
             snakemake.params["clustering"],
-            snakemake.params['exclude_carriers'],
+            snakemake.params["exclude_carriers"],
             snakemake.output,
             aggregation_strategies=aggregation_strategies,
         )
@@ -579,9 +621,9 @@ if __name__ == "__main__":
         n, cluster_map = cluster(
             n,
             int(snakemake.wildcards.simpl),
-            snakemake.params['focus_weights'],
-            snakemake.params['renewable'],
-            snakemake.params['solver_name'],
+            snakemake.params["focus_weights"],
+            snakemake.params["renewable"],
+            snakemake.params["solver_name"],
             cluster_param.get("algorithm", "hac"),
             cluster_param.get("feature", None),
             aggregation_strategies,

From d2f6138ca564d3286fcd8f3f93e954d22b4d16d7 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 22 May 2023 21:56:04 +0000
Subject: [PATCH 110/176] [pre-commit.ci] pre-commit autoupdate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

updates:
- [github.com/PyCQA/docformatter: v1.6.5 → v1.7.1](https://github.com/PyCQA/docformatter/compare/v1.6.5...v1.7.1)
---
 .pre-commit-config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 48915f4c..34d13746 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -39,7 +39,7 @@ repos:
 
   # Make docstrings PEP 257 compliant
 - repo: https://github.com/PyCQA/docformatter
-  rev: v1.6.5
+  rev: v1.7.1
   hooks:
   - id: docformatter
     args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"]

From 0f09545d9626b7e64c883a721f592847c5fd4367 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 22 May 2023 21:56:29 +0000
Subject: [PATCH 111/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/_helpers.py         | 9 +++++----
 scripts/build_retro_cost.py | 4 ++--
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/scripts/_helpers.py b/scripts/_helpers.py
index a67fb105..8086c1b5 100644
--- a/scripts/_helpers.py
+++ b/scripts/_helpers.py
@@ -385,10 +385,11 @@ def mock_snakemake(rulename, configfiles=[], **wildcards):
 
 
 def override_component_attrs(directory):
-    """Tell PyPSA that links can have multiple outputs by
-    overriding the component_attrs. This can be done for
-    as many buses as you need with format busi for i = 2,3,4,5,....
-    See https://pypsa.org/doc/components.html#link-with-multiple-outputs-or-inputs
+    """
+    Tell PyPSA that links can have multiple outputs by overriding the
+    component_attrs. This can be done for as many buses as you need with format
+    busi for i = 2,3,4,5,.... See https://pypsa.org/doc/components.html#link-
+    with-multiple-outputs-or-inputs.
 
     Parameters
     ----------
diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py
index 9dbfc375..63ae0743 100644
--- a/scripts/build_retro_cost.py
+++ b/scripts/build_retro_cost.py
@@ -698,8 +698,8 @@ def get_solar_gains_per_year(window_area):
 
 def map_to_lstrength(l_strength, df):
     """
-    renames column names from a pandas dataframe to map tabula retrofitting
-    strengths [2 = moderate, 3 = ambitious] to l_strength
+    Renames column names from a pandas dataframe to map tabula retrofitting
+    strengths [2 = moderate, 3 = ambitious] to l_strength.
     """
     middle = len(l_strength) // 2
     map_to_l = pd.MultiIndex.from_arrays(

From 312dd81f216f2ab55ee2eb8b13c5a54532fe9e06 Mon Sep 17 00:00:00 2001
From: Koen van Greevenbroek <koen.v.greevenbroek@uit.no>
Date: Wed, 24 May 2023 11:13:37 +0200
Subject: [PATCH 112/176] Fix bug with underground H2 storage creation

For some small model regions, none of the H2 cavern types specified in
the configuration might actually be available, in which case the line
`h2_caverns = h2_caverns[cavern_types].sum(axis=1)` throws an error.
---
 scripts/prepare_sector_network.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 012c9714..57deb5fd 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -1105,7 +1105,11 @@ def add_storage_and_grids(n, costs):
     cavern_types = snakemake.config["sector"]["hydrogen_underground_storage_locations"]
     h2_caverns = pd.read_csv(snakemake.input.h2_cavern, index_col=0)
 
-    if not h2_caverns.empty and options["hydrogen_underground_storage"]:
+    if (
+        not h2_caverns.empty
+        and options["hydrogen_underground_storage"]
+        and set(cavern_types).intersection(h2_caverns.columns)
+    ):
         h2_caverns = h2_caverns[cavern_types].sum(axis=1)
 
         # only use sites with at least 2 TWh potential

From dc043d8aeff58455ad0e339485de375e8afb8d02 Mon Sep 17 00:00:00 2001
From: lucie_rc <104382956+LucieRC@users.noreply.github.com>
Date: Wed, 24 May 2023 15:46:53 +0300
Subject: [PATCH 113/176] Update Discord link README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 02cc3163..79cdfa65 100644
--- a/README.md
+++ b/README.md
@@ -96,7 +96,7 @@ repository](https://doi.org/10.5281/zenodo.3601881).
 We strongly welcome anyone interested in contributing to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on GitHub.
 -   In case of code-related **questions**, please post on [stack overflow](https://stackoverflow.com/questions/tagged/pypsa).
 -   For non-programming related and more general questions please refer to the [mailing list](https://groups.google.com/group/pypsa).
--   To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.gg/JTdvaEBb).
+-   To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.com/invite/AnuJBk23FU).
 -   For **bugs and feature requests**, please use the [PyPSA-Eur Github Issues page](https://github.com/PyPSA/pypsa-eur/issues).
 
 # Licence

From 1c73bb03426beaaaedc10df7ee83d8f036395a4d Mon Sep 17 00:00:00 2001
From: virio-andreyana <arsyan.virio@gmail.com>
Date: Sat, 27 May 2023 12:22:53 +0200
Subject: [PATCH 114/176] add params for solve_network

---
 rules/solve_electricity.smk |  6 ++++++
 rules/solve_myopic.smk      |  6 ++++++
 rules/solve_overnight.smk   |  6 ++++++
 scripts/solve_network.py    | 38 ++++++++++++++++++-------------------
 4 files changed, 37 insertions(+), 19 deletions(-)

diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk
index 2e9c4b44..f0ce55c8 100644
--- a/rules/solve_electricity.smk
+++ b/rules/solve_electricity.smk
@@ -6,6 +6,12 @@
 rule solve_network:
     params:
         solving=config["solving"],
+        config_parts={
+                    "foresight":config["foresight"],
+                    "planning_horizons":config["scenario"]["planning_horizons"],
+                    "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200),
+                    "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"]
+                    },
     input:
         network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
     output:
diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk
index ec7638cf..c682f780 100644
--- a/rules/solve_myopic.smk
+++ b/rules/solve_myopic.smk
@@ -84,6 +84,12 @@ ruleorder: add_existing_baseyear > add_brownfield
 rule solve_sector_network_myopic:
     params:
         solving=config["solving"],
+        config_parts={
+                    "foresight":config["foresight"],
+                    "planning_horizons":config["scenario"]["planning_horizons"],
+                    "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200),
+                    "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"]
+                    },
     input:
         overrides="data/override_component_attrs",
         network=RESULTS
diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk
index b657eb2b..050372bd 100644
--- a/rules/solve_overnight.smk
+++ b/rules/solve_overnight.smk
@@ -6,6 +6,12 @@
 rule solve_sector_network:
     params:
         solving=config["solving"],
+        config_parts={
+                    "foresight":config["foresight"],
+                    "planning_horizons":config["scenario"]["planning_horizons"],
+                    "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200),
+                    "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"]
+                    },
     input:
         overrides="data/override_component_attrs",
         network=RESULTS
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index 180c83c6..d615fc85 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -44,14 +44,14 @@ pypsa.pf.logger.setLevel(logging.WARNING)
 from pypsa.descriptors import get_switchable_as_dense as get_as_dense
 
 
-def add_land_use_constraint(n, config):
+def add_land_use_constraint(n, param, config):
     if "m" in snakemake.wildcards.clusters:
-        _add_land_use_constraint_m(n, config)
+        _add_land_use_constraint_m(n, param, config)
     else:
-        _add_land_use_constraint(n, config)
+        _add_land_use_constraint(n, param)
 
 
-def _add_land_use_constraint(n, config):
+def _add_land_use_constraint(n, param):
     # warning: this will miss existing offwind which is not classed AC-DC and has carrier 'offwind'
 
     for carrier in ["solar", "onwind", "offwind-ac", "offwind-dc"]:
@@ -80,10 +80,10 @@ def _add_land_use_constraint(n, config):
     n.generators.p_nom_max.clip(lower=0, inplace=True)
 
 
-def _add_land_use_constraint_m(n, config):
+def _add_land_use_constraint_m(n, param, config):
     # if generators clustering is lower than network clustering, land_use accounting is at generators clusters
 
-    planning_horizons = config["scenario"]["planning_horizons"]
+    planning_horizons = param["planning_horizons"]
     grouping_years = config["existing_capacities"]["grouping_years"]
     current_horizon = snakemake.wildcards.planning_horizons
 
@@ -141,7 +141,7 @@ def add_co2_sequestration_limit(n, limit=200):
     )
 
 
-def prepare_network(n, solve_opts=None, config=None):
+def prepare_network(n, solve_opts=None, config=None, param=None):
     if "clip_p_max_pu" in solve_opts:
         for df in (
             n.generators_t.p_max_pu,
@@ -191,11 +191,11 @@ def prepare_network(n, solve_opts=None, config=None):
         n.set_snapshots(n.snapshots[:nhours])
         n.snapshot_weightings[:] = 8760.0 / nhours
 
-    if config["foresight"] == "myopic":
-        add_land_use_constraint(n, config)
+    if param["foresight"] == "myopic":
+        add_land_use_constraint(n, param, config)
 
     if n.stores.carrier.eq("co2 stored").any():
-        limit = config["sector"].get("co2_sequestration_potential", 200)
+        limit = param["co2_sequestration_potential"]
         add_co2_sequestration_limit(n, limit=limit)
 
     return n
@@ -222,7 +222,7 @@ def add_CCL_constraints(n, config):
         agg_p_nom_limits: data/agg_p_nom_minmax.csv
     """
     agg_p_nom_minmax = pd.read_csv(
-        config["electricity"]["agg_p_nom_limits"], index_col=[0, 1]
+        config['electricity']["agg_p_nom_limits"], index_col=[0, 1]
     )
     logger.info("Adding generation capacity constraints per carrier and country")
     p_nom = n.model["Generator-p_nom"]
@@ -370,7 +370,7 @@ def add_SAFE_constraints(n, config):
     Which sets a reserve margin of 10% above the peak demand.
     """
     peakdemand = n.loads_t.p_set.sum(axis=1).max()
-    margin = 1.0 + config["electricity"]["SAFE_reservemargin"]
+    margin = 1.0 + config['electricity']["SAFE_reservemargin"]
     reserve_margin = peakdemand * margin
     # TODO: do not take this from the plotting config!
     conv_techs = config["plotting"]["conv_techs"]
@@ -590,13 +590,13 @@ def extra_functionality(n, snapshots):
     add_pipe_retrofit_constraint(n)
 
 
-def solve_network(n, config, opts="", **kwargs):
-    set_of_options = config["solving"]["solver"]["options"]
+def solve_network(n, config, solving_param, opts="", **kwargs):
+    set_of_options = solving_param["solver"]["options"]
     solver_options = (
-        config["solving"]["solver_options"][set_of_options] if set_of_options else {}
+        solving_param["solver_options"][set_of_options] if set_of_options else {}
     )
-    solver_name = config["solving"]["solver"]["name"]
-    cf_solving = config["solving"]["options"]
+    solver_name = solving_param["solver"]["name"]
+    cf_solving = solving_param["options"]
     track_iterations = cf_solving.get("track_iterations", False)
     min_iterations = cf_solving.get("min_iterations", 4)
     max_iterations = cf_solving.get("max_iterations", 6)
@@ -675,10 +675,10 @@ if __name__ == "__main__":
     else:
         n = pypsa.Network(snakemake.input.network)
 
-    n = prepare_network(n, solve_opts, config=snakemake.config)
+    n = prepare_network(n, solve_opts, config=snakemake.config, param=snakemake.params["config_parts"])
 
     n = solve_network(
-        n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver
+        n, config=snakemake.config, solving_param=snakemake.params["solving"], opts=opts, log_fn=snakemake.log.solver
     )
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))

From ead87afce1ab0d7179b9012f88df1089e609bc61 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sat, 27 May 2023 10:23:28 +0000
Subject: [PATCH 115/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 rules/solve_electricity.smk | 14 +++++++++-----
 rules/solve_myopic.smk      | 14 +++++++++-----
 rules/solve_overnight.smk   | 14 +++++++++-----
 scripts/solve_network.py    | 14 ++++++++++----
 4 files changed, 37 insertions(+), 19 deletions(-)

diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk
index f0ce55c8..30520dc1 100644
--- a/rules/solve_electricity.smk
+++ b/rules/solve_electricity.smk
@@ -7,11 +7,15 @@ rule solve_network:
     params:
         solving=config["solving"],
         config_parts={
-                    "foresight":config["foresight"],
-                    "planning_horizons":config["scenario"]["planning_horizons"],
-                    "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200),
-                    "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"]
-                    },
+            "foresight": config["foresight"],
+            "planning_horizons": config["scenario"]["planning_horizons"],
+            "co2_sequestration_potential": config["sector"].get(
+                "co2_sequestration_potential", 200
+            ),
+            "H2_retrofit_capacity_per_CH4": config["sector"][
+            "H2_retrofit_capacity_per_CH4"
+            ],
+        },
     input:
         network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
     output:
diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk
index c682f780..22555f75 100644
--- a/rules/solve_myopic.smk
+++ b/rules/solve_myopic.smk
@@ -85,11 +85,15 @@ rule solve_sector_network_myopic:
     params:
         solving=config["solving"],
         config_parts={
-                    "foresight":config["foresight"],
-                    "planning_horizons":config["scenario"]["planning_horizons"],
-                    "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200),
-                    "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"]
-                    },
+            "foresight": config["foresight"],
+            "planning_horizons": config["scenario"]["planning_horizons"],
+            "co2_sequestration_potential": config["sector"].get(
+                "co2_sequestration_potential", 200
+            ),
+            "H2_retrofit_capacity_per_CH4": config["sector"][
+            "H2_retrofit_capacity_per_CH4"
+            ],
+        },
     input:
         overrides="data/override_component_attrs",
         network=RESULTS
diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk
index 050372bd..a523f132 100644
--- a/rules/solve_overnight.smk
+++ b/rules/solve_overnight.smk
@@ -7,11 +7,15 @@ rule solve_sector_network:
     params:
         solving=config["solving"],
         config_parts={
-                    "foresight":config["foresight"],
-                    "planning_horizons":config["scenario"]["planning_horizons"],
-                    "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200),
-                    "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"]
-                    },
+            "foresight": config["foresight"],
+            "planning_horizons": config["scenario"]["planning_horizons"],
+            "co2_sequestration_potential": config["sector"].get(
+                "co2_sequestration_potential", 200
+            ),
+            "H2_retrofit_capacity_per_CH4": config["sector"][
+            "H2_retrofit_capacity_per_CH4"
+            ],
+        },
     input:
         overrides="data/override_component_attrs",
         network=RESULTS
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index d615fc85..6e07e340 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -222,7 +222,7 @@ def add_CCL_constraints(n, config):
         agg_p_nom_limits: data/agg_p_nom_minmax.csv
     """
     agg_p_nom_minmax = pd.read_csv(
-        config['electricity']["agg_p_nom_limits"], index_col=[0, 1]
+        config["electricity"]["agg_p_nom_limits"], index_col=[0, 1]
     )
     logger.info("Adding generation capacity constraints per carrier and country")
     p_nom = n.model["Generator-p_nom"]
@@ -370,7 +370,7 @@ def add_SAFE_constraints(n, config):
     Which sets a reserve margin of 10% above the peak demand.
     """
     peakdemand = n.loads_t.p_set.sum(axis=1).max()
-    margin = 1.0 + config['electricity']["SAFE_reservemargin"]
+    margin = 1.0 + config["electricity"]["SAFE_reservemargin"]
     reserve_margin = peakdemand * margin
     # TODO: do not take this from the plotting config!
     conv_techs = config["plotting"]["conv_techs"]
@@ -675,10 +675,16 @@ if __name__ == "__main__":
     else:
         n = pypsa.Network(snakemake.input.network)
 
-    n = prepare_network(n, solve_opts, config=snakemake.config, param=snakemake.params["config_parts"])
+    n = prepare_network(
+        n, solve_opts, config=snakemake.config, param=snakemake.params["config_parts"]
+    )
 
     n = solve_network(
-        n, config=snakemake.config, solving_param=snakemake.params["solving"], opts=opts, log_fn=snakemake.log.solver
+        n,
+        config=snakemake.config,
+        solving_param=snakemake.params["solving"],
+        opts=opts,
+        log_fn=snakemake.log.solver,
     )
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))

From e581ca930c24de2c3e3fa4d0d6bdac0fbfa06b30 Mon Sep 17 00:00:00 2001
From: virio-andreyana <arsyan.virio@gmail.com>
Date: Sat, 27 May 2023 15:50:37 +0200
Subject: [PATCH 116/176] made params more specific

---
 rules/build_electricity.smk                  | 29 +++++---
 rules/build_sector.smk                       | 72 ++++++++++++++++----
 rules/postprocess.smk                        |  3 +-
 rules/solve_electricity.smk                  |  2 +-
 rules/solve_myopic.smk                       |  7 +-
 scripts/add_brownfield.py                    |  6 +-
 scripts/add_electricity.py                   | 11 ++-
 scripts/add_existing_baseyear.py             |  2 +-
 scripts/add_extra_components.py              | 23 +++----
 scripts/build_cop_profiles.py                |  2 +-
 scripts/build_cutout.py                      |  2 +-
 scripts/build_hydro_profile.py               |  2 +-
 scripts/build_industrial_distribution_key.py |  2 +-
 scripts/build_industry_sector_ratios.py      |  2 +-
 scripts/build_powerplants.py                 |  4 +-
 scripts/build_retro_cost.py                  |  2 +-
 scripts/build_sequestration_potentials.py    |  2 +-
 scripts/cluster_network.py                   |  8 +--
 scripts/plot_summary.py                      |  4 +-
 scripts/prepare_network.py                   | 10 +--
 scripts/prepare_sector_network.py            | 14 ++--
 scripts/simplify_network.py                  |  2 +-
 scripts/solve_operations_network.py          |  2 +-
 23 files changed, 131 insertions(+), 82 deletions(-)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index 45436c5d..9b3a03fb 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -39,7 +39,8 @@ rule build_electricity_demand:
 
 rule build_powerplants:
     params:
-        electricity=config["electricity"],
+        powerplants_filter=config["electricity"]["powerplants_filter"],
+        custom_powerplants=config["electricity"]["custom_powerplants"],
         countries=config["countries"],
     input:
         base_network=RESOURCES + "networks/base.nc",
@@ -138,7 +139,7 @@ if config["enable"].get("build_cutout", False):
     rule build_cutout:
         params:
             snapshots=config["snapshots"],
-            atlite=config["atlite"],
+            cutouts=config["atlite"]["cutouts"],
         input:
             regions_onshore=RESOURCES + "regions_onshore.geojson",
             regions_offshore=RESOURCES + "regions_offshore.geojson",
@@ -252,6 +253,7 @@ rule build_renewable_profiles:
 
 rule build_hydro_profile:
     params:
+        hydro=config["renewable"]["hydro"],
         countries=config["countries"],
         renewable=config["renewable"],
     input:
@@ -272,8 +274,8 @@ rule build_hydro_profile:
 
 rule add_electricity:
     params:
-        lines=config["lines"],
-        load=config["load"],
+        length_factor=config["lines"]["length_factor"],
+        scaling_factor=config["load"]["scaling_factor"],
         countries=config["countries"],
         renewable=config["renewable"],
         electricity=config["electricity"],
@@ -316,7 +318,7 @@ rule add_electricity:
 rule simplify_network:
     params:
         clustering=config["clustering"],
-        electricity=config["electricity"],
+        max_hours=config["electricity"]["max_hours"],
         costs=config["costs"],
         renewable=config["renewable"],
         length_factor=config["lines"]["length_factor"],
@@ -352,10 +354,11 @@ rule simplify_network:
 
 rule cluster_network:
     params:
-        solving=config["solving"],
-        electricity=config["electricity"],
+        solver_name=config["solving"]["solver"]["name"],
+        max_hours=config["electricity"]["max_hours"],
+        conventional_carriers=config["electricity"].get("conventional_carriers", []),
         costs=config["costs"],
-        lines=config["lines"],
+        length_factor=config["lines"]["length_factor"],
         renewable=config["renewable"],
         clustering=config["clustering"],
         custom_busmap=config["enable"].get("custom_busmap", False),
@@ -392,7 +395,8 @@ rule cluster_network:
 rule add_extra_components:
     params:
         costs=config["costs"],
-        electricity=config["electricity"],
+        ext_carriers=config["electricity"]["extendable_carriers"],
+        max_hours=config["electricity"]["max_hours"],
     input:
         network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc",
         tech_costs=COSTS,
@@ -414,9 +418,12 @@ rule add_extra_components:
 rule prepare_network:
     params:
         links=config["links"],
-        solving=config["solving"],
         lines=config["lines"],
-        electricity=config["electricity"],
+        solver_name=config["solving"]["solver"]["name"],
+        co2base=config["electricity"]["co2base"],
+        co2limit=config["electricity"]["co2limit"],
+        gaslimit=config["electricity"].get("gaslimit"),
+        max_hours=config["electricity"]["max_hours"],
         costs=config["costs"],
     input:
         RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc",
diff --git a/rules/build_sector.smk b/rules/build_sector.smk
index 9faae4b9..da8a60f9 100644
--- a/rules/build_sector.smk
+++ b/rules/build_sector.smk
@@ -186,7 +186,7 @@ rule build_temperature_profiles:
 
 rule build_cop_profiles:
     params:
-        sector=config["sector"],
+        heat_pump_sink_T=config["sector"]["heat_pump_sink_T"],
     input:
         temp_soil_total=RESOURCES + "temp_soil_total_elec_s{simpl}_{clusters}.nc",
         temp_soil_rural=RESOURCES + "temp_soil_rural_elec_s{simpl}_{clusters}.nc",
@@ -330,7 +330,7 @@ if config["sector"]["regional_co2_sequestration_potential"]["enable"]:
 
     rule build_sequestration_potentials:
         params:
-            sector=config["sector"],
+            co2seq_potential=config["sector"]["regional_co2_sequestration_potential"],
         input:
             sequestration_potential=HTTP.remote(
                 "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson",
@@ -405,8 +405,27 @@ rule build_ammonia_production:
 
 rule build_industry_sector_ratios:
     params:
-        industry=config["industry"],
-        sector=config["sector"],
+        industry={
+            "H2_DRI":config["industry"]["H2_DRI"],
+            "elec_DRI":config["industry"]["elec_DRI"],
+            "HVC_production_today":config["industry"]["HVC_production_today"],
+            "petrochemical_process_emissions":config["industry"]["petrochemical_process_emissions"],
+            "NH3_process_emissions":config["industry"]["NH3_process_emissions"],
+            "MWh_CH4_per_tNH3_SMR":config["industry"]["MWh_CH4_per_tNH3_SMR"],
+            "MWh_elec_per_tNH3_SMR":config["industry"]["MWh_elec_per_tNH3_SMR"],
+            "chlorine_production_today":config["industry"]["chlorine_production_today"],
+            "MWh_H2_per_tCl":config["industry"]["MWh_H2_per_tCl"],
+            "MWh_elec_per_tCl":config["industry"]["MWh_elec_per_tCl"],
+            "methanol_production_today":config["industry"]["methanol_production_today"],
+            "MWh_CH4_per_tMeOH":config["industry"]["MWh_CH4_per_tMeOH"],
+            "MWh_elec_per_tMeOH":config["industry"]["MWh_elec_per_tMeOH"],
+            "MWh_elec_per_tHVC_mechanical_recycling":config["industry"]["MWh_elec_per_tHVC_mechanical_recycling"],
+            "MWh_elec_per_tHVC_chemical_recycling":config["industry"]["MWh_elec_per_tHVC_chemical_recycling"],
+            "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"],
+            "MWh_H2_per_tNH3_electrolysis":config["industry"]["MWh_H2_per_tNH3_electrolysis"],
+            "MWh_elec_per_tNH3_electrolysis":config["industry"]["MWh_elec_per_tNH3_electrolysis"]
+        },
+        sector_amonia=config["sector"].get("ammonia", False),
     input:
         ammonia_production=RESOURCES + "ammonia_production.csv",
         idees="data/jrc-idees-2015",
@@ -427,7 +446,12 @@ rule build_industry_sector_ratios:
 
 rule build_industrial_production_per_country:
     params:
-        industry=config["industry"],
+        industry={
+            "reference_year":config["industry"]["reference_year"],
+            "HVC_production_today":config["industry"]["HVC_production_today"],
+            "chlorine_production_today":config["industry"]["chlorine_production_today"],
+            "methanol_production_today":config["industry"]["methanol_production_today"]
+        },
         countries=config["countries"],
     input:
         ammonia_production=RESOURCES + "ammonia_production.csv",
@@ -451,7 +475,14 @@ rule build_industrial_production_per_country:
 
 rule build_industrial_production_per_country_tomorrow:
     params:
-        industry=config["industry"],
+        industry={
+            "St_primary_fraction":config["industry"]["St_primary_fraction"],
+            "DRI_fraction":config["industry"]["DRI_fraction"],
+            "Al_primary_fraction":config["industry"]["Al_primary_fraction"],
+            "HVC_mechanical_recycling_fraction":config["industry"]["HVC_mechanical_recycling_fraction"],
+            "HVC_chemical_recycling_fraction":config["industry"]["HVC_chemical_recycling_fraction"],
+            "HVC_primary_fraction":config["industry"]["HVC_primary_fraction"]
+        },
     input:
         industrial_production_per_country=RESOURCES
         + "industrial_production_per_country.csv",
@@ -477,7 +508,7 @@ rule build_industrial_production_per_country_tomorrow:
 
 rule build_industrial_distribution_key:
     params:
-        industry=config["industry"],
+        hotmaps_locate_missing=config["industry"].get("hotmaps_locate_missing", False),
         countries=config["countries"],
     input:
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
@@ -555,7 +586,12 @@ rule build_industrial_energy_demand_per_node:
 rule build_industrial_energy_demand_per_country_today:
     params:
         countries=config["countries"],
-        industry=config["industry"],
+        industry={
+            "reference_year":config["industry"].get("reference_year", 2015),
+            "MWh_CH4_per_tNH3_SMR":config["industry"]["MWh_CH4_per_tNH3_SMR"],
+            "MWh_elec_per_tNH3_SMR":config["industry"]["MWh_elec_per_tNH3_SMR"],
+            "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"]
+        },
     input:
         jrc="data/jrc-idees-2015",
         ammonia_production=RESOURCES + "ammonia_production.csv",
@@ -603,7 +639,7 @@ if config["sector"]["retrofitting"]["retro_endogen"]:
 
     rule build_retro_cost:
         params:
-            sector=config["sector"],
+            retrofitting=config["sector"]["retrofitting"],
             countries=config["countries"],
         input:
             building_stock="data/retro/data_building_stock.csv",
@@ -705,17 +741,23 @@ rule build_transport_demand:
 rule prepare_sector_network:
     params:
         co2_budget=config["co2_budget"],
-        solving=config["solving"],
-        existing_capacities=config["existing_capacities"],
+        solver_name=config["solving"]["solver"]["name"],
+        conventional_carriers=config["existing_capacities"]["conventional_carriers"],
         foresight=config["foresight"],
         costs=config["costs"],
         sector=config["sector"],
-        industry=config["industry"],
+        industry={
+            "MWh_elec_per_tNH3_electrolysis":config["industry"]["MWh_elec_per_tNH3_electrolysis"],
+            "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"],
+            "MWh_H2_per_tNH3_electrolysis":config["industry"]["MWh_H2_per_tNH3_electrolysis"],
+            "MWh_NH3_per_MWh_H2_cracker":config["industry"]["MWh_NH3_per_MWh_H2_cracker"]
+        },
         pypsa_eur=config["pypsa_eur"],
-        lines=config["lines"],
-        scenario=config["scenario"],
+        length_factor=config["lines"]["length_factor"],
+        planning_horizons=config["scenario"]["planning_horizons"],
         countries=config["countries"],
-        energy=config["energy"],
+        emissions_scope=config["energy"]["emissions"],
+        report_year=config["energy"]["eurostat_report_year"],
         RDIR=RDIR,
     input:
         **build_retro_cost_output,
diff --git a/rules/postprocess.smk b/rules/postprocess.smk
index 9eb04ef5..ac80cd10 100644
--- a/rules/postprocess.smk
+++ b/rules/postprocess.smk
@@ -122,7 +122,8 @@ rule make_summary:
 rule plot_summary:
     params:
         countries=config["countries"],
-        scenario=config["scenario"],
+        planning_horizons=config["scenario"]["planning_horizons"],
+        sector_opts=config["scenario"]["sector_opts"],
         plotting=config["plotting"],
         RDIR=RDIR,
     input:
diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk
index 30520dc1..892415c4 100644
--- a/rules/solve_electricity.smk
+++ b/rules/solve_electricity.smk
@@ -41,7 +41,7 @@ rule solve_network:
 
 rule solve_operations_network:
     params:
-        solving=config["solving"],
+        options=config["solving"]["options"],
     input:
         network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
     output:
diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk
index 22555f75..8ec1c4a8 100644
--- a/rules/solve_myopic.smk
+++ b/rules/solve_myopic.smk
@@ -5,7 +5,7 @@
 
 rule add_existing_baseyear:
     params:
-        scenario=config["scenario"],
+        baseyear=config["scenario"]["planning_horizons"][0],
         sector=config["sector"],
         existing_capacities=config["existing_capacities"],
         costs=config["costs"],
@@ -48,8 +48,9 @@ rule add_existing_baseyear:
 
 rule add_brownfield:
     params:
-        sector=config["sector"],
-        existing_capacities=config["existing_capacities"],
+        H2_retrofit=config["sector"]["H2_retrofit"],
+        H2_retrofit_capacity_per_CH4=config["sector"]["H2_retrofit_capacity_per_CH4"],
+        threshold_capacity=config["existing_capacities"]["threshold_capacity"],
     input:
         overrides="data/override_component_attrs",
         network=RESULTS
diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py
index 9330953b..48dffbb9 100644
--- a/scripts/add_brownfield.py
+++ b/scripts/add_brownfield.py
@@ -49,7 +49,7 @@ def add_brownfield(n, n_p, year):
             )
         ]
 
-        threshold = snakemake.params["existing_capacities"]["threshold_capacity"]
+        threshold = snakemake.params["threshold_capacity"]
 
         if not chp_heat.empty:
             threshold_chp_heat = (
@@ -87,7 +87,7 @@ def add_brownfield(n, n_p, year):
 
         # deal with gas network
         pipe_carrier = ["gas pipeline"]
-        if snakemake.params["sector"]["H2_retrofit"]:
+        if snakemake.params["H2_retrofit"]:
             # drop capacities of previous year to avoid duplicating
             to_drop = n.links.carrier.isin(pipe_carrier) & (n.links.build_year != year)
             n.mremove("Link", n.links.loc[to_drop].index)
@@ -98,7 +98,7 @@ def add_brownfield(n, n_p, year):
                 & (n.links.build_year != year)
             ].index
             gas_pipes_i = n.links[n.links.carrier.isin(pipe_carrier)].index
-            CH4_per_H2 = 1 / snakemake.params["sector"]["H2_retrofit_capacity_per_CH4"]
+            CH4_per_H2 = 1 / snakemake.params["H2_retrofit_capacity_per_CH4"]
             fr = "H2 pipeline retrofitted"
             to = "gas pipeline"
             # today's pipe capacity
diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 69d91b87..dc5fb7cc 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -137,7 +137,7 @@ def _add_missing_carriers_from_costs(n, costs, carriers):
     n.import_components_from_dataframe(emissions, "Carrier")
 
 
-def load_costs(tech_costs, params, elec_params, Nyears=1.0):
+def load_costs(tech_costs, params, max_hours, Nyears=1.0):
     # set all asset costs and other parameters
     costs = pd.read_csv(tech_costs, index_col=[0, 1]).sort_index()
 
@@ -180,7 +180,6 @@ def load_costs(tech_costs, params, elec_params, Nyears=1.0):
             dict(capital_cost=capital_cost, marginal_cost=0.0, co2_emissions=0.0)
         )
 
-    max_hours = elec_params["max_hours"]
     costs.loc["battery"] = costs_for_storage(
         costs.loc["battery storage"],
         costs.loc["battery inverter"],
@@ -728,7 +727,7 @@ if __name__ == "__main__":
     costs = load_costs(
         snakemake.input.tech_costs,
         snakemake.params["costs"],
-        snakemake.params["electricity"],
+        snakemake.params["electricity"]["max_hours"],
         Nyears,
     )
     ppl = load_powerplants(snakemake.input.powerplants)
@@ -759,10 +758,10 @@ if __name__ == "__main__":
         snakemake.input.load,
         snakemake.input.nuts3_shapes,
         snakemake.params["countries"],
-        snakemake.params["load"]["scaling_factor"],
+        snakemake.params["scaling_factor"],
     )
 
-    update_transmission_costs(n, costs, snakemake.params["lines"]["length_factor"])
+    update_transmission_costs(n, costs, snakemake.params["length_factor"])
 
     conventional_inputs = {
         k: v for k, v in snakemake.input.items() if k.startswith("conventional_")
@@ -783,7 +782,7 @@ if __name__ == "__main__":
         snakemake.input,
         renewable_carriers,
         extendable_carriers,
-        snakemake.params["lines"]["length_factor"],
+        snakemake.params["length_factor"],
     )
 
     if "hydro" in renewable_carriers:
diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py
index a4d3748b..bc1d3a05 100644
--- a/scripts/add_existing_baseyear.py
+++ b/scripts/add_existing_baseyear.py
@@ -615,7 +615,7 @@ if __name__ == "__main__":
     options = snakemake.params["sector"]
     opts = snakemake.wildcards.sector_opts.split("-")
 
-    baseyear = snakemake.params["scenario"]["planning_horizons"][0]
+    baseyear = snakemake.params["baseyear"]
 
     overrides = override_component_attrs(snakemake.input.overrides)
     n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py
index 08178c0a..ce4e533e 100644
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@@ -67,9 +67,8 @@ idx = pd.IndexSlice
 logger = logging.getLogger(__name__)
 
 
-def attach_storageunits(n, costs, elec_opts):
-    carriers = elec_opts["extendable_carriers"]["StorageUnit"]
-    max_hours = elec_opts["max_hours"]
+def attach_storageunits(n, costs, ext_carriers, max_hours):
+    carriers = ext_carriers["StorageUnit"]
 
     _add_missing_carriers_from_costs(n, costs, carriers)
 
@@ -99,8 +98,8 @@ def attach_storageunits(n, costs, elec_opts):
         )
 
 
-def attach_stores(n, costs, elec_opts):
-    carriers = elec_opts["extendable_carriers"]["Store"]
+def attach_stores(n, costs, ext_carriers):
+    carriers = ext_carriers["Store"]
 
     _add_missing_carriers_from_costs(n, costs, carriers)
 
@@ -187,8 +186,7 @@ def attach_stores(n, costs, elec_opts):
         )
 
 
-def attach_hydrogen_pipelines(n, costs, elec_opts):
-    ext_carriers = elec_opts["extendable_carriers"]
+def attach_hydrogen_pipelines(n, costs, ext_carriers):
     as_stores = ext_carriers.get("Store", [])
 
     if "H2 pipeline" not in ext_carriers.get("Link", []):
@@ -235,16 +233,17 @@ if __name__ == "__main__":
     configure_logging(snakemake)
 
     n = pypsa.Network(snakemake.input.network)
-    elec_config = snakemake.params["electricity"]
+    ext_carriers = snakemake.params["ext_carriers"]
+    max_hours = snakemake.params["max_hours"]
 
     Nyears = n.snapshot_weightings.objective.sum() / 8760.0
     costs = load_costs(
-        snakemake.input.tech_costs, snakemake.params["costs"], elec_config, Nyears
+        snakemake.input.tech_costs, snakemake.params["costs"], max_hours, Nyears
     )
 
-    attach_storageunits(n, costs, elec_config)
-    attach_stores(n, costs, elec_config)
-    attach_hydrogen_pipelines(n, costs, elec_config)
+    attach_storageunits(n, costs, ext_carriers, max_hours)
+    attach_stores(n, costs, ext_carriers)
+    attach_hydrogen_pipelines(n, costs, ext_carriers)
 
     add_nice_carrier_names(n, snakemake.config)
 
diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py
index 7128ec0d..983eda2d 100644
--- a/scripts/build_cop_profiles.py
+++ b/scripts/build_cop_profiles.py
@@ -39,7 +39,7 @@ if __name__ == "__main__":
         for source in ["air", "soil"]:
             source_T = xr.open_dataarray(snakemake.input[f"temp_{source}_{area}"])
 
-            delta_T = snakemake.params["sector"]["heat_pump_sink_T"] - source_T
+            delta_T = snakemake.params["heat_pump_sink_T"] - source_T
 
             cop = coefficient_of_performance(delta_T, source)
 
diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py
index 2f61f017..798588d9 100644
--- a/scripts/build_cutout.py
+++ b/scripts/build_cutout.py
@@ -106,7 +106,7 @@ if __name__ == "__main__":
         snakemake = mock_snakemake("build_cutout", cutout="europe-2013-era5")
     configure_logging(snakemake)
 
-    cutout_params = snakemake.params["atlite"]["cutouts"][snakemake.wildcards.cutout]
+    cutout_params = snakemake.params["cutouts"][snakemake.wildcards.cutout]
 
     snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"])
     time = [snapshots[0], snapshots[-1]]
diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py
index dd686be3..26bf31c6 100644
--- a/scripts/build_hydro_profile.py
+++ b/scripts/build_hydro_profile.py
@@ -130,7 +130,7 @@ if __name__ == "__main__":
         snakemake = mock_snakemake("build_hydro_profile")
     configure_logging(snakemake)
 
-    params_hydro = snakemake.params["renewable"]["hydro"]
+    params_hydro = snakemake.params["hydro"]
     cutout = atlite.Cutout(snakemake.input.cutout)
 
     countries = snakemake.params["countries"]
diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py
index 9a513673..a51ff8b2 100644
--- a/scripts/build_industrial_distribution_key.py
+++ b/scripts/build_industrial_distribution_key.py
@@ -73,7 +73,7 @@ def prepare_hotmaps_database(regions):
 
     df[["srid", "coordinates"]] = df.geom.str.split(";", expand=True)
 
-    if snakemake.params["industry"].get("hotmaps_locate_missing", False):
+    if snakemake.params["hotmaps_locate_missing"]:
         df = locate_missing_industrial_sites(df)
 
     # remove those sites without valid locations
diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py
index 2ec007a9..fa9e5c18 100644
--- a/scripts/build_industry_sector_ratios.py
+++ b/scripts/build_industry_sector_ratios.py
@@ -439,7 +439,7 @@ def chemicals_industry():
 
     sector = "Ammonia"
     df[sector] = 0.0
-    if snakemake.params["sector"].get("ammonia", False):
+    if snakemake.params["sector_amonia"]:
         df.loc["ammonia", sector] = params["MWh_NH3_per_tNH3"]
     else:
         df.loc["hydrogen", sector] = params["MWh_H2_per_tNH3_electrolysis"]
diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py
index bd0ee74e..6edd4ac4 100755
--- a/scripts/build_powerplants.py
+++ b/scripts/build_powerplants.py
@@ -134,12 +134,12 @@ if __name__ == "__main__":
     ppl = ppl.query('not (Country in @available_countries and Fueltype == "Bioenergy")')
     ppl = pd.concat([ppl, opsd])
 
-    ppl_query = snakemake.params["electricity"]["powerplants_filter"]
+    ppl_query = snakemake.params["powerplants_filter"]
     if isinstance(ppl_query, str):
         ppl.query(ppl_query, inplace=True)
 
     # add carriers from own powerplant files:
-    custom_ppl_query = snakemake.params["electricity"]["custom_powerplants"]
+    custom_ppl_query = snakemake.params["custom_powerplants"]
     ppl = add_custom_powerplants(
         ppl, snakemake.input.custom_powerplants, custom_ppl_query
     )
diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py
index c52d4eb6..8473be7a 100644
--- a/scripts/build_retro_cost.py
+++ b/scripts/build_retro_cost.py
@@ -1040,7 +1040,7 @@ if __name__ == "__main__":
 
     #  ********  config  *********************************************************
 
-    retro_opts = snakemake.params["sector"]["retrofitting"]
+    retro_opts = snakemake.params["retrofitting"]
     interest_rate = retro_opts["interest_rate"]
     annualise_cost = retro_opts["annualise_cost"]  # annualise the investment costs
     tax_weighting = retro_opts[
diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py
index 5c388b2e..9d26b0b9 100644
--- a/scripts/build_sequestration_potentials.py
+++ b/scripts/build_sequestration_potentials.py
@@ -41,7 +41,7 @@ if __name__ == "__main__":
             "build_sequestration_potentials", simpl="", clusters="181"
         )
 
-    cf = snakemake.params["sector"]["regional_co2_sequestration_potential"]
+    cf = snakemake.params["co2seq_potential"]
 
     gdf = gpd.read_file(snakemake.input.sequestration_potential[0])
 
diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py
index d217e5ed..3cfbd214 100644
--- a/scripts/cluster_network.py
+++ b/scripts/cluster_network.py
@@ -479,7 +479,7 @@ if __name__ == "__main__":
     if snakemake.wildcards.clusters.endswith("m"):
         n_clusters = int(snakemake.wildcards.clusters[:-1])
         conventional = set(
-            snakemake.params["electricity"].get("conventional_carriers", [])
+            snakemake.params["conventional_carriers"]
         )
         aggregate_carriers = conventional.intersection(aggregate_carriers)
     elif snakemake.wildcards.clusters == "all":
@@ -495,13 +495,13 @@ if __name__ == "__main__":
             n, busmap, linemap, linemap, pd.Series(dtype="O")
         )
     else:
-        line_length_factor = snakemake.params["lines"]["length_factor"]
+        line_length_factor = snakemake.params["length_factor"]
         Nyears = n.snapshot_weightings.objective.sum() / 8760
 
         hvac_overhead_cost = load_costs(
             snakemake.input.tech_costs,
             snakemake.params["costs"],
-            snakemake.params["electricity"],
+            snakemake.params["max_hours"],
             Nyears,
         ).at["HVAC overhead", "capital_cost"]
 
@@ -539,7 +539,7 @@ if __name__ == "__main__":
             aggregate_carriers,
             line_length_factor,
             aggregation_strategies,
-            snakemake.params["solving"]["solver"]["name"],
+            snakemake.params["solver_name"],
             cluster_config.get("algorithm", "hac"),
             cluster_config.get("feature", "solar+onwind-time"),
             hvac_overhead_cost,
diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py
index 4aa37de5..36f75207 100644
--- a/scripts/plot_summary.py
+++ b/scripts/plot_summary.py
@@ -455,7 +455,7 @@ def plot_carbon_budget_distribution(input_eurostat):
     ax1 = plt.subplot(gs1[0, 0])
     ax1.set_ylabel("CO$_2$ emissions (Gt per year)", fontsize=22)
     ax1.set_ylim([0, 5])
-    ax1.set_xlim([1990, snakemake.params["scenario"]["planning_horizons"][-1] + 1])
+    ax1.set_xlim([1990, snakemake.params["planning_horizons"][-1] + 1])
 
     path_cb = "results/" + snakemake.params.RDIR + "/csvs/"
     countries = snakemake.params["countries"]
@@ -555,7 +555,7 @@ if __name__ == "__main__":
 
     plot_balances()
 
-    for sector_opts in snakemake.params["scenario"]["sector_opts"]:
+    for sector_opts in snakemake.params["sector_opts"]:
         opts = sector_opts.split("-")
         for o in opts:
             if "cb" in o:
diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index 46e4e74a..51777ef5 100755
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -254,7 +254,7 @@ if __name__ == "__main__":
     costs = load_costs(
         snakemake.input.tech_costs,
         snakemake.params["costs"],
-        snakemake.params["electricity"],
+        snakemake.params["max_hours"],
         Nyears,
     )
 
@@ -269,7 +269,7 @@ if __name__ == "__main__":
     for o in opts:
         m = re.match(r"^\d+seg$", o, re.IGNORECASE)
         if m is not None:
-            solver_name = snakemake.params["solving"]["solver"]["name"]
+            solver_name = snakemake.params["solver_name"]
             n = apply_time_segmentation(n, m.group(0)[:-3], solver_name)
             break
 
@@ -277,11 +277,11 @@ if __name__ == "__main__":
         if "Co2L" in o:
             m = re.findall("[0-9]*\.?[0-9]+$", o)
             if len(m) > 0:
-                co2limit = float(m[0]) * snakemake.params["electricity"]["co2base"]
+                co2limit = float(m[0]) * snakemake.params["co2base"]
                 add_co2limit(n, co2limit, Nyears)
                 logger.info("Setting CO2 limit according to wildcard value.")
             else:
-                add_co2limit(n, snakemake.params["electricity"]["co2limit"], Nyears)
+                add_co2limit(n, snakemake.params["co2limit"], Nyears)
                 logger.info("Setting CO2 limit according to config value.")
             break
 
@@ -293,7 +293,7 @@ if __name__ == "__main__":
                 add_gaslimit(n, limit, Nyears)
                 logger.info("Setting gas usage limit according to wildcard value.")
             else:
-                add_gaslimit(n, snakemake.params["electricity"].get("gaslimit"), Nyears)
+                add_gaslimit(n, snakemake.params["gaslimit"], Nyears)
                 logger.info("Setting gas usage limit according to config value.")
             break
 
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 6a71b1e2..54ab1f94 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -252,7 +252,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year):
         countries, input_eurostat, opts, emissions_scope, report_year, year=2018
     )
 
-    planning_horizons = snakemake.params["scenario"]["planning_horizons"]
+    planning_horizons = snakemake.params["planning_horizons"]
     t_0 = planning_horizons[0]
 
     if "be" in o:
@@ -391,7 +391,7 @@ def update_wind_solar_costs(n, costs):
         with xr.open_dataset(profile) as ds:
             underwater_fraction = ds["underwater_fraction"].to_pandas()
             connection_cost = (
-                snakemake.params["lines"]["length_factor"]
+                snakemake.params["length_factor"]
                 * ds["average_distance"].to_pandas()
                 * (
                     underwater_fraction
@@ -3300,7 +3300,7 @@ if __name__ == "__main__":
     if snakemake.params["foresight"] == "myopic":
         add_lifetime_wind_solar(n, costs)
 
-        conventional = snakemake.params["existing_capacities"]["conventional_carriers"]
+        conventional = snakemake.params["conventional_carriers"]
         for carrier in conventional:
             add_carrier_buses(n, carrier)
 
@@ -3365,7 +3365,7 @@ if __name__ == "__main__":
     if options["allam_cycle"]:
         add_allam(n, costs)
 
-    solver_name = snakemake.params["solving"]["solver"]["name"]
+    solver_name = snakemake.params["solver_name"]
     n = set_temporal_aggregation(n, opts, solver_name)
 
     limit_type = "config"
@@ -3376,8 +3376,8 @@ if __name__ == "__main__":
         limit_type = "carbon budget"
         fn = "results/" + snakemake.params.RDIR + "/csvs/carbon_budget_distribution.csv"
         if not os.path.exists(fn):
-            emissions_scope = snakemake.params["energy"]["emissions"]
-            report_year = snakemake.params["energy"]["eurostat_report_year"]
+            emissions_scope = snakemake.params["emissions_scope"]
+            report_year = snakemake.params["report_year"]
             build_carbon_budget(
                 o, snakemake.input.eurostat, fn, emissions_scope, report_year
             )
@@ -3413,7 +3413,7 @@ if __name__ == "__main__":
         add_electricity_grid_connection(n, costs)
 
     first_year_myopic = (snakemake.params["foresight"] == "myopic") and (
-        snakemake.params["scenario"]["planning_horizons"][0] == investment_year
+        snakemake.params["planning_horizons"][0] == investment_year
     )
 
     if options.get("cluster_heat_buses", False) and not first_year_myopic:
diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 689e8084..83e932a3 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -560,7 +560,7 @@ if __name__ == "__main__":
     technology_costs = load_costs(
         snakemake.input.tech_costs,
         snakemake.params["costs"],
-        snakemake.params["electricity"],
+        snakemake.params["max_hours"],
         Nyears,
     )
 
diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py
index c268e2ee..421bc515 100644
--- a/scripts/solve_operations_network.py
+++ b/scripts/solve_operations_network.py
@@ -41,7 +41,7 @@ if __name__ == "__main__":
 
     opts = (snakemake.wildcards.opts + "-" + snakemake.wildcards.sector_opts).split("-")
     opts = [o for o in opts if o != ""]
-    solve_opts = snakemake.params["solving"]["options"]
+    solve_opts = snakemake.params["options"]
 
     np.random.seed(solve_opts.get("seed", 123))
 

From f3e8fe23122bb47e2d8dfb4c8264e920007ee20b Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sat, 27 May 2023 13:50:58 +0000
Subject: [PATCH 117/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 rules/build_sector.smk     | 100 ++++++++++++++++++++++++-------------
 scripts/cluster_network.py |   4 +-
 2 files changed, 65 insertions(+), 39 deletions(-)

diff --git a/rules/build_sector.smk b/rules/build_sector.smk
index da8a60f9..2d6c8263 100644
--- a/rules/build_sector.smk
+++ b/rules/build_sector.smk
@@ -406,24 +406,38 @@ rule build_ammonia_production:
 rule build_industry_sector_ratios:
     params:
         industry={
-            "H2_DRI":config["industry"]["H2_DRI"],
-            "elec_DRI":config["industry"]["elec_DRI"],
-            "HVC_production_today":config["industry"]["HVC_production_today"],
-            "petrochemical_process_emissions":config["industry"]["petrochemical_process_emissions"],
-            "NH3_process_emissions":config["industry"]["NH3_process_emissions"],
-            "MWh_CH4_per_tNH3_SMR":config["industry"]["MWh_CH4_per_tNH3_SMR"],
-            "MWh_elec_per_tNH3_SMR":config["industry"]["MWh_elec_per_tNH3_SMR"],
-            "chlorine_production_today":config["industry"]["chlorine_production_today"],
-            "MWh_H2_per_tCl":config["industry"]["MWh_H2_per_tCl"],
-            "MWh_elec_per_tCl":config["industry"]["MWh_elec_per_tCl"],
-            "methanol_production_today":config["industry"]["methanol_production_today"],
-            "MWh_CH4_per_tMeOH":config["industry"]["MWh_CH4_per_tMeOH"],
-            "MWh_elec_per_tMeOH":config["industry"]["MWh_elec_per_tMeOH"],
-            "MWh_elec_per_tHVC_mechanical_recycling":config["industry"]["MWh_elec_per_tHVC_mechanical_recycling"],
-            "MWh_elec_per_tHVC_chemical_recycling":config["industry"]["MWh_elec_per_tHVC_chemical_recycling"],
-            "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"],
-            "MWh_H2_per_tNH3_electrolysis":config["industry"]["MWh_H2_per_tNH3_electrolysis"],
-            "MWh_elec_per_tNH3_electrolysis":config["industry"]["MWh_elec_per_tNH3_electrolysis"]
+            "H2_DRI": config["industry"]["H2_DRI"],
+            "elec_DRI": config["industry"]["elec_DRI"],
+            "HVC_production_today": config["industry"]["HVC_production_today"],
+            "petrochemical_process_emissions": config["industry"][
+            "petrochemical_process_emissions"
+            ],
+            "NH3_process_emissions": config["industry"]["NH3_process_emissions"],
+            "MWh_CH4_per_tNH3_SMR": config["industry"]["MWh_CH4_per_tNH3_SMR"],
+            "MWh_elec_per_tNH3_SMR": config["industry"]["MWh_elec_per_tNH3_SMR"],
+            "chlorine_production_today": config["industry"][
+            "chlorine_production_today"
+            ],
+            "MWh_H2_per_tCl": config["industry"]["MWh_H2_per_tCl"],
+            "MWh_elec_per_tCl": config["industry"]["MWh_elec_per_tCl"],
+            "methanol_production_today": config["industry"][
+            "methanol_production_today"
+            ],
+            "MWh_CH4_per_tMeOH": config["industry"]["MWh_CH4_per_tMeOH"],
+            "MWh_elec_per_tMeOH": config["industry"]["MWh_elec_per_tMeOH"],
+            "MWh_elec_per_tHVC_mechanical_recycling": config["industry"][
+            "MWh_elec_per_tHVC_mechanical_recycling"
+            ],
+            "MWh_elec_per_tHVC_chemical_recycling": config["industry"][
+            "MWh_elec_per_tHVC_chemical_recycling"
+            ],
+            "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"],
+            "MWh_H2_per_tNH3_electrolysis": config["industry"][
+            "MWh_H2_per_tNH3_electrolysis"
+            ],
+            "MWh_elec_per_tNH3_electrolysis": config["industry"][
+            "MWh_elec_per_tNH3_electrolysis"
+            ],
         },
         sector_amonia=config["sector"].get("ammonia", False),
     input:
@@ -447,10 +461,14 @@ rule build_industry_sector_ratios:
 rule build_industrial_production_per_country:
     params:
         industry={
-            "reference_year":config["industry"]["reference_year"],
-            "HVC_production_today":config["industry"]["HVC_production_today"],
-            "chlorine_production_today":config["industry"]["chlorine_production_today"],
-            "methanol_production_today":config["industry"]["methanol_production_today"]
+            "reference_year": config["industry"]["reference_year"],
+            "HVC_production_today": config["industry"]["HVC_production_today"],
+            "chlorine_production_today": config["industry"][
+            "chlorine_production_today"
+            ],
+            "methanol_production_today": config["industry"][
+            "methanol_production_today"
+            ],
         },
         countries=config["countries"],
     input:
@@ -476,12 +494,16 @@ rule build_industrial_production_per_country:
 rule build_industrial_production_per_country_tomorrow:
     params:
         industry={
-            "St_primary_fraction":config["industry"]["St_primary_fraction"],
-            "DRI_fraction":config["industry"]["DRI_fraction"],
-            "Al_primary_fraction":config["industry"]["Al_primary_fraction"],
-            "HVC_mechanical_recycling_fraction":config["industry"]["HVC_mechanical_recycling_fraction"],
-            "HVC_chemical_recycling_fraction":config["industry"]["HVC_chemical_recycling_fraction"],
-            "HVC_primary_fraction":config["industry"]["HVC_primary_fraction"]
+            "St_primary_fraction": config["industry"]["St_primary_fraction"],
+            "DRI_fraction": config["industry"]["DRI_fraction"],
+            "Al_primary_fraction": config["industry"]["Al_primary_fraction"],
+            "HVC_mechanical_recycling_fraction": config["industry"][
+            "HVC_mechanical_recycling_fraction"
+            ],
+            "HVC_chemical_recycling_fraction": config["industry"][
+            "HVC_chemical_recycling_fraction"
+            ],
+            "HVC_primary_fraction": config["industry"]["HVC_primary_fraction"],
         },
     input:
         industrial_production_per_country=RESOURCES
@@ -587,10 +609,10 @@ rule build_industrial_energy_demand_per_country_today:
     params:
         countries=config["countries"],
         industry={
-            "reference_year":config["industry"].get("reference_year", 2015),
-            "MWh_CH4_per_tNH3_SMR":config["industry"]["MWh_CH4_per_tNH3_SMR"],
-            "MWh_elec_per_tNH3_SMR":config["industry"]["MWh_elec_per_tNH3_SMR"],
-            "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"]
+            "reference_year": config["industry"].get("reference_year", 2015),
+            "MWh_CH4_per_tNH3_SMR": config["industry"]["MWh_CH4_per_tNH3_SMR"],
+            "MWh_elec_per_tNH3_SMR": config["industry"]["MWh_elec_per_tNH3_SMR"],
+            "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"],
         },
     input:
         jrc="data/jrc-idees-2015",
@@ -747,10 +769,16 @@ rule prepare_sector_network:
         costs=config["costs"],
         sector=config["sector"],
         industry={
-            "MWh_elec_per_tNH3_electrolysis":config["industry"]["MWh_elec_per_tNH3_electrolysis"],
-            "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"],
-            "MWh_H2_per_tNH3_electrolysis":config["industry"]["MWh_H2_per_tNH3_electrolysis"],
-            "MWh_NH3_per_MWh_H2_cracker":config["industry"]["MWh_NH3_per_MWh_H2_cracker"]
+            "MWh_elec_per_tNH3_electrolysis": config["industry"][
+            "MWh_elec_per_tNH3_electrolysis"
+            ],
+            "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"],
+            "MWh_H2_per_tNH3_electrolysis": config["industry"][
+            "MWh_H2_per_tNH3_electrolysis"
+            ],
+            "MWh_NH3_per_MWh_H2_cracker": config["industry"][
+            "MWh_NH3_per_MWh_H2_cracker"
+            ],
         },
         pypsa_eur=config["pypsa_eur"],
         length_factor=config["lines"]["length_factor"],
diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py
index 3cfbd214..b86bb225 100644
--- a/scripts/cluster_network.py
+++ b/scripts/cluster_network.py
@@ -478,9 +478,7 @@ if __name__ == "__main__":
     aggregate_carriers = set(n.generators.carrier) - set(exclude_carriers)
     if snakemake.wildcards.clusters.endswith("m"):
         n_clusters = int(snakemake.wildcards.clusters[:-1])
-        conventional = set(
-            snakemake.params["conventional_carriers"]
-        )
+        conventional = set(snakemake.params["conventional_carriers"])
         aggregate_carriers = conventional.intersection(aggregate_carriers)
     elif snakemake.wildcards.clusters == "all":
         n_clusters = len(n.buses)

From b67a00903af6b5d55019da50bd845a217e07d628 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Wed, 31 May 2023 10:54:27 +0200
Subject: [PATCH 118/176] using higher keys for industries

---
 rules/build_electricity.smk |  1 -
 rules/build_sector.smk      | 79 +++----------------------------------
 2 files changed, 5 insertions(+), 75 deletions(-)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index 9b3a03fb..34276ba4 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -255,7 +255,6 @@ rule build_hydro_profile:
     params:
         hydro=config["renewable"]["hydro"],
         countries=config["countries"],
-        renewable=config["renewable"],
     input:
         country_shapes=RESOURCES + "country_shapes.geojson",
         eia_hydro_generation="data/eia_hydro_annual_generation.csv",
diff --git a/rules/build_sector.smk b/rules/build_sector.smk
index 2d6c8263..3714cbad 100644
--- a/rules/build_sector.smk
+++ b/rules/build_sector.smk
@@ -405,40 +405,7 @@ rule build_ammonia_production:
 
 rule build_industry_sector_ratios:
     params:
-        industry={
-            "H2_DRI": config["industry"]["H2_DRI"],
-            "elec_DRI": config["industry"]["elec_DRI"],
-            "HVC_production_today": config["industry"]["HVC_production_today"],
-            "petrochemical_process_emissions": config["industry"][
-            "petrochemical_process_emissions"
-            ],
-            "NH3_process_emissions": config["industry"]["NH3_process_emissions"],
-            "MWh_CH4_per_tNH3_SMR": config["industry"]["MWh_CH4_per_tNH3_SMR"],
-            "MWh_elec_per_tNH3_SMR": config["industry"]["MWh_elec_per_tNH3_SMR"],
-            "chlorine_production_today": config["industry"][
-            "chlorine_production_today"
-            ],
-            "MWh_H2_per_tCl": config["industry"]["MWh_H2_per_tCl"],
-            "MWh_elec_per_tCl": config["industry"]["MWh_elec_per_tCl"],
-            "methanol_production_today": config["industry"][
-            "methanol_production_today"
-            ],
-            "MWh_CH4_per_tMeOH": config["industry"]["MWh_CH4_per_tMeOH"],
-            "MWh_elec_per_tMeOH": config["industry"]["MWh_elec_per_tMeOH"],
-            "MWh_elec_per_tHVC_mechanical_recycling": config["industry"][
-            "MWh_elec_per_tHVC_mechanical_recycling"
-            ],
-            "MWh_elec_per_tHVC_chemical_recycling": config["industry"][
-            "MWh_elec_per_tHVC_chemical_recycling"
-            ],
-            "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"],
-            "MWh_H2_per_tNH3_electrolysis": config["industry"][
-            "MWh_H2_per_tNH3_electrolysis"
-            ],
-            "MWh_elec_per_tNH3_electrolysis": config["industry"][
-            "MWh_elec_per_tNH3_electrolysis"
-            ],
-        },
+        industry=config["industry"],
         sector_amonia=config["sector"].get("ammonia", False),
     input:
         ammonia_production=RESOURCES + "ammonia_production.csv",
@@ -460,16 +427,7 @@ rule build_industry_sector_ratios:
 
 rule build_industrial_production_per_country:
     params:
-        industry={
-            "reference_year": config["industry"]["reference_year"],
-            "HVC_production_today": config["industry"]["HVC_production_today"],
-            "chlorine_production_today": config["industry"][
-            "chlorine_production_today"
-            ],
-            "methanol_production_today": config["industry"][
-            "methanol_production_today"
-            ],
-        },
+        industry=config["industry"],
         countries=config["countries"],
     input:
         ammonia_production=RESOURCES + "ammonia_production.csv",
@@ -493,18 +451,7 @@ rule build_industrial_production_per_country:
 
 rule build_industrial_production_per_country_tomorrow:
     params:
-        industry={
-            "St_primary_fraction": config["industry"]["St_primary_fraction"],
-            "DRI_fraction": config["industry"]["DRI_fraction"],
-            "Al_primary_fraction": config["industry"]["Al_primary_fraction"],
-            "HVC_mechanical_recycling_fraction": config["industry"][
-            "HVC_mechanical_recycling_fraction"
-            ],
-            "HVC_chemical_recycling_fraction": config["industry"][
-            "HVC_chemical_recycling_fraction"
-            ],
-            "HVC_primary_fraction": config["industry"]["HVC_primary_fraction"],
-        },
+        industry=config["industry"],
     input:
         industrial_production_per_country=RESOURCES
         + "industrial_production_per_country.csv",
@@ -608,12 +555,7 @@ rule build_industrial_energy_demand_per_node:
 rule build_industrial_energy_demand_per_country_today:
     params:
         countries=config["countries"],
-        industry={
-            "reference_year": config["industry"].get("reference_year", 2015),
-            "MWh_CH4_per_tNH3_SMR": config["industry"]["MWh_CH4_per_tNH3_SMR"],
-            "MWh_elec_per_tNH3_SMR": config["industry"]["MWh_elec_per_tNH3_SMR"],
-            "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"],
-        },
+        industry=config["industry"],
     input:
         jrc="data/jrc-idees-2015",
         ammonia_production=RESOURCES + "ammonia_production.csv",
@@ -768,18 +710,7 @@ rule prepare_sector_network:
         foresight=config["foresight"],
         costs=config["costs"],
         sector=config["sector"],
-        industry={
-            "MWh_elec_per_tNH3_electrolysis": config["industry"][
-            "MWh_elec_per_tNH3_electrolysis"
-            ],
-            "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"],
-            "MWh_H2_per_tNH3_electrolysis": config["industry"][
-            "MWh_H2_per_tNH3_electrolysis"
-            ],
-            "MWh_NH3_per_MWh_H2_cracker": config["industry"][
-            "MWh_NH3_per_MWh_H2_cracker"
-            ],
-        },
+        industry=config["industry"],
         pypsa_eur=config["pypsa_eur"],
         length_factor=config["lines"]["length_factor"],
         planning_horizons=config["scenario"]["planning_horizons"],

From 83ff639ad57f40f7a61027951843cd7e4b334642 Mon Sep 17 00:00:00 2001
From: virio-andreyana <arsyan.virio@gmail.com>
Date: Fri, 2 Jun 2023 12:52:49 +0200
Subject: [PATCH 119/176] amend solve_network

---
 rules/solve_electricity.smk | 13 +++----------
 rules/solve_myopic.smk      | 13 +++----------
 rules/solve_overnight.smk   | 13 +++----------
 scripts/solve_network.py    | 25 +++++++++++++++----------
 4 files changed, 24 insertions(+), 40 deletions(-)

diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk
index 892415c4..e7b264f7 100644
--- a/rules/solve_electricity.smk
+++ b/rules/solve_electricity.smk
@@ -6,16 +6,9 @@
 rule solve_network:
     params:
         solving=config["solving"],
-        config_parts={
-            "foresight": config["foresight"],
-            "planning_horizons": config["scenario"]["planning_horizons"],
-            "co2_sequestration_potential": config["sector"].get(
-                "co2_sequestration_potential", 200
-            ),
-            "H2_retrofit_capacity_per_CH4": config["sector"][
-            "H2_retrofit_capacity_per_CH4"
-            ],
-        },
+        foresight=config["foresight"],
+        planning_horizons=config["scenario"]["planning_horizons"],
+        co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200),
     input:
         network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
     output:
diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk
index 8ec1c4a8..39567b92 100644
--- a/rules/solve_myopic.smk
+++ b/rules/solve_myopic.smk
@@ -85,16 +85,9 @@ ruleorder: add_existing_baseyear > add_brownfield
 rule solve_sector_network_myopic:
     params:
         solving=config["solving"],
-        config_parts={
-            "foresight": config["foresight"],
-            "planning_horizons": config["scenario"]["planning_horizons"],
-            "co2_sequestration_potential": config["sector"].get(
-                "co2_sequestration_potential", 200
-            ),
-            "H2_retrofit_capacity_per_CH4": config["sector"][
-            "H2_retrofit_capacity_per_CH4"
-            ],
-        },
+        foresight=config["foresight"],
+        planning_horizons=config["scenario"]["planning_horizons"],
+        co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200),
     input:
         overrides="data/override_component_attrs",
         network=RESULTS
diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk
index a523f132..71ef075e 100644
--- a/rules/solve_overnight.smk
+++ b/rules/solve_overnight.smk
@@ -6,16 +6,9 @@
 rule solve_sector_network:
     params:
         solving=config["solving"],
-        config_parts={
-            "foresight": config["foresight"],
-            "planning_horizons": config["scenario"]["planning_horizons"],
-            "co2_sequestration_potential": config["sector"].get(
-                "co2_sequestration_potential", 200
-            ),
-            "H2_retrofit_capacity_per_CH4": config["sector"][
-            "H2_retrofit_capacity_per_CH4"
-            ],
-        },
+        foresight=config["foresight"],
+        planning_horizons=config["scenario"]["planning_horizons"],
+        co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200),
     input:
         overrides="data/override_component_attrs",
         network=RESULTS
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index 6e07e340..90ff3a2e 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -44,14 +44,14 @@ pypsa.pf.logger.setLevel(logging.WARNING)
 from pypsa.descriptors import get_switchable_as_dense as get_as_dense
 
 
-def add_land_use_constraint(n, param, config):
+def add_land_use_constraint(n, planning_horizons_param, config):
     if "m" in snakemake.wildcards.clusters:
-        _add_land_use_constraint_m(n, param, config)
+        _add_land_use_constraint_m(n, planning_horizons_param, config)
     else:
-        _add_land_use_constraint(n, param)
+        _add_land_use_constraint(n)
 
 
-def _add_land_use_constraint(n, param):
+def _add_land_use_constraint(n):
     # warning: this will miss existing offwind which is not classed AC-DC and has carrier 'offwind'
 
     for carrier in ["solar", "onwind", "offwind-ac", "offwind-dc"]:
@@ -80,7 +80,7 @@ def _add_land_use_constraint(n, param):
     n.generators.p_nom_max.clip(lower=0, inplace=True)
 
 
-def _add_land_use_constraint_m(n, param, config):
+def _add_land_use_constraint_m(n, planning_horizons_param, config):
     # if generators clustering is lower than network clustering, land_use accounting is at generators clusters
 
     planning_horizons = param["planning_horizons"]
@@ -141,7 +141,7 @@ def add_co2_sequestration_limit(n, limit=200):
     )
 
 
-def prepare_network(n, solve_opts=None, config=None, param=None):
+def prepare_network(n, solve_opts=None, config=None, foresight_param=None, planning_horizons_param=None, co2_sequestration_potential=None):
     if "clip_p_max_pu" in solve_opts:
         for df in (
             n.generators_t.p_max_pu,
@@ -191,11 +191,11 @@ def prepare_network(n, solve_opts=None, config=None, param=None):
         n.set_snapshots(n.snapshots[:nhours])
         n.snapshot_weightings[:] = 8760.0 / nhours
 
-    if param["foresight"] == "myopic":
-        add_land_use_constraint(n, param, config)
+    if foresight_param == "myopic":
+        add_land_use_constraint(n, planning_horizons_param, config)
 
     if n.stores.carrier.eq("co2 stored").any():
-        limit = param["co2_sequestration_potential"]
+        limit = co2_sequestration_potential_param
         add_co2_sequestration_limit(n, limit=limit)
 
     return n
@@ -676,7 +676,12 @@ if __name__ == "__main__":
         n = pypsa.Network(snakemake.input.network)
 
     n = prepare_network(
-        n, solve_opts, config=snakemake.config, param=snakemake.params["config_parts"]
+        n,
+        solve_opts,
+        config=snakemake.config,
+        foresight_param=snakemake.params["foresight"],
+        planning_horizons_param=snakemake.params["planning_horizons"],
+        co2_sequestration_potential=snakemake.params["co2_sequestration_potential"]
     )
 
     n = solve_network(

From 0f5bb8b893afc4955f608d9a206035f63e2bb6fe Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 2 Jun 2023 10:53:09 +0000
Subject: [PATCH 120/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 rules/solve_electricity.smk |  4 +++-
 rules/solve_myopic.smk      |  4 +++-
 rules/solve_overnight.smk   |  4 +++-
 scripts/solve_network.py    | 11 +++++++++--
 4 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk
index e7b264f7..f73a5b0c 100644
--- a/rules/solve_electricity.smk
+++ b/rules/solve_electricity.smk
@@ -8,7 +8,9 @@ rule solve_network:
         solving=config["solving"],
         foresight=config["foresight"],
         planning_horizons=config["scenario"]["planning_horizons"],
-        co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200),
+        co2_sequestration_potential=config["sector"].get(
+            "co2_sequestration_potential", 200
+        ),
     input:
         network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
     output:
diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk
index 39567b92..086375c2 100644
--- a/rules/solve_myopic.smk
+++ b/rules/solve_myopic.smk
@@ -87,7 +87,9 @@ rule solve_sector_network_myopic:
         solving=config["solving"],
         foresight=config["foresight"],
         planning_horizons=config["scenario"]["planning_horizons"],
-        co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200),
+        co2_sequestration_potential=config["sector"].get(
+            "co2_sequestration_potential", 200
+        ),
     input:
         overrides="data/override_component_attrs",
         network=RESULTS
diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk
index 71ef075e..fc430f4d 100644
--- a/rules/solve_overnight.smk
+++ b/rules/solve_overnight.smk
@@ -8,7 +8,9 @@ rule solve_sector_network:
         solving=config["solving"],
         foresight=config["foresight"],
         planning_horizons=config["scenario"]["planning_horizons"],
-        co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200),
+        co2_sequestration_potential=config["sector"].get(
+            "co2_sequestration_potential", 200
+        ),
     input:
         overrides="data/override_component_attrs",
         network=RESULTS
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index 90ff3a2e..ab1175ab 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -141,7 +141,14 @@ def add_co2_sequestration_limit(n, limit=200):
     )
 
 
-def prepare_network(n, solve_opts=None, config=None, foresight_param=None, planning_horizons_param=None, co2_sequestration_potential=None):
+def prepare_network(
+    n,
+    solve_opts=None,
+    config=None,
+    foresight_param=None,
+    planning_horizons_param=None,
+    co2_sequestration_potential=None,
+):
     if "clip_p_max_pu" in solve_opts:
         for df in (
             n.generators_t.p_max_pu,
@@ -681,7 +688,7 @@ if __name__ == "__main__":
         config=snakemake.config,
         foresight_param=snakemake.params["foresight"],
         planning_horizons_param=snakemake.params["planning_horizons"],
-        co2_sequestration_potential=snakemake.params["co2_sequestration_potential"]
+        co2_sequestration_potential=snakemake.params["co2_sequestration_potential"],
     )
 
     n = solve_network(

From 8dbcca560cd47d58d098dd91e751efd45314b8e7 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Sat, 3 Jun 2023 12:09:51 +0200
Subject: [PATCH 121/176] minor fixes in solve_network.py

---
 scripts/solve_network.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index ab1175ab..ea5446ad 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -147,7 +147,7 @@ def prepare_network(
     config=None,
     foresight_param=None,
     planning_horizons_param=None,
-    co2_sequestration_potential=None,
+    co2_sequestration_potential_param=None,
 ):
     if "clip_p_max_pu" in solve_opts:
         for df in (
@@ -688,7 +688,7 @@ if __name__ == "__main__":
         config=snakemake.config,
         foresight_param=snakemake.params["foresight"],
         planning_horizons_param=snakemake.params["planning_horizons"],
-        co2_sequestration_potential=snakemake.params["co2_sequestration_potential"],
+        co2_sequestration_potential_param=snakemake.params["co2_sequestration_potential"],
     )
 
     n = solve_network(

From d6057cb92be6fdb07d02da23e13035b90a57d1fc Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sat, 3 Jun 2023 10:10:06 +0000
Subject: [PATCH 122/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/solve_network.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index ea5446ad..41679a03 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -688,7 +688,9 @@ if __name__ == "__main__":
         config=snakemake.config,
         foresight_param=snakemake.params["foresight"],
         planning_horizons_param=snakemake.params["planning_horizons"],
-        co2_sequestration_potential_param=snakemake.params["co2_sequestration_potential"],
+        co2_sequestration_potential_param=snakemake.params[
+            "co2_sequestration_potential"
+        ],
     )
 
     n = solve_network(

From 176bebdef0f12de8c639c83e70a9b949ae5c318e Mon Sep 17 00:00:00 2001
From: Davide Fioriti <67809479+davide-f@users.noreply.github.com>
Date: Fri, 9 Jun 2023 17:18:25 +0200
Subject: [PATCH 123/176] Update cluster_network to avoid adding deleted links
 in clustered network

---
 scripts/cluster_network.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py
index 7572d3b3..d5d27ef7 100644
--- a/scripts/cluster_network.py
+++ b/scripts/cluster_network.py
@@ -424,7 +424,10 @@ def clustering_for_n_clusters(
             n.links.eval("underwater_fraction * length").div(nc.links.length).dropna()
         )
         nc.links["capital_cost"] = nc.links["capital_cost"].add(
-            (nc.links.length - n.links.length).clip(lower=0).mul(extended_link_costs),
+            (nc.links.length - n.links.length)
+            .clip(lower=0)
+            .mul(extended_link_costs)
+            .dropna(),
             fill_value=0,
         )
 

From e44f8d66cdb2abb62a9ef59f1141b1895ce8d167 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 12 Jun 2023 22:22:08 +0000
Subject: [PATCH 124/176] [pre-commit.ci] pre-commit autoupdate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

updates:
- [github.com/PyCQA/docformatter: v1.7.1 → v1.7.2](https://github.com/PyCQA/docformatter/compare/v1.7.1...v1.7.2)
---
 .pre-commit-config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 34d13746..826a4819 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -39,7 +39,7 @@ repos:
 
   # Make docstrings PEP 257 compliant
 - repo: https://github.com/PyCQA/docformatter
-  rev: v1.7.1
+  rev: v1.7.2
   hooks:
   - id: docformatter
     args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"]

From 12b93562453ccce90df248c2686040c984a1612b Mon Sep 17 00:00:00 2001
From: virio-andreyana <arsyan.virio@gmail.com>
Date: Wed, 14 Jun 2023 11:03:07 +0200
Subject: [PATCH 125/176] add info to release_notes

---
 doc/release_notes.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index 3af16477..8f26ac02 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -10,6 +10,7 @@ Release Notes
 Upcoming Release
 ================
 
+* ``param:`` section in rule definition are added to track changed settings in ``config.yaml``. The goal is to automatically re-execute rules whose parameters have changed. See `Non-file parameters for rules <https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#non-file-parameters-for-rules>`_ in the snakemake documentation.
 
 * **Important:** The configuration files are now located in the ``config`` directory. This counts for ``config.default.yaml``, ``config.yaml`` as well as the test configuration files which are now located in ``config/test``. Config files that are still in the root directory will be ignored.
 

From 45cac01ea3c49529c4b68714e783bedb60565f00 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Wed, 14 Jun 2023 13:28:20 +0200
Subject: [PATCH 126/176] adjust param accessing in cluster, simplify and
 build_elec script

---
 rules/build_electricity.smk         |  24 ++---
 scripts/add_electricity.py          |  40 ++++-----
 scripts/build_renewable_profiles.py |   2 +-
 scripts/cluster_network.py          |  56 +++++-------
 scripts/simplify_network.py         | 131 ++++++++++++----------------
 5 files changed, 109 insertions(+), 144 deletions(-)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index 34276ba4..fd66cc04 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -316,16 +316,14 @@ rule add_electricity:
 
 rule simplify_network:
     params:
-        clustering=config["clustering"],
-        max_hours=config["electricity"]["max_hours"],
+        simplify_network=config["clustering"]["simplify_network"],
+        aggregation_strategies=config["clustering"].get("aggregation_strategies", {}),
+        focus_weights=config.get("focus_weights", None),
+        renewable_carriers=config["electricity"]["renewable_carriers"],
         costs=config["costs"],
-        renewable=config["renewable"],
+        max_hours=config["electricity"]["max_hours"],
         length_factor=config["lines"]["length_factor"],
         p_max_pu=config["links"].get("p_max_pu", 1.0),
-        exclude_carriers=config["clustering"]["simplify_network"].get(
-            "exclude_carriers", []
-        ),
-        focus_weights=config.get("focus_weights", None),
         solver_name=config["solving"]["solver"]["name"],
     input:
         network=RESOURCES + "networks/elec.nc",
@@ -353,14 +351,16 @@ rule simplify_network:
 
 rule cluster_network:
     params:
-        solver_name=config["solving"]["solver"]["name"],
-        max_hours=config["electricity"]["max_hours"],
+        cluster_network=config["clustering"]["cluster_network"],
+        aggregation_strategies=config["clustering"].get("aggregation_strategies", {}),
+        custom_busmap=config["enable"].get("custom_busmap", False),
+        focus_weights=config.get("focus_weights", None),
+        renewable_carriers=config["electricity"]["renewable_carriers"],
         conventional_carriers=config["electricity"].get("conventional_carriers", []),
         costs=config["costs"],
+        max_hours=config["electricity"]["max_hours"],
         length_factor=config["lines"]["length_factor"],
-        renewable=config["renewable"],
-        clustering=config["clustering"],
-        custom_busmap=config["enable"].get("custom_busmap", False),
+        solver_name=config["solving"]["solver"]["name"],
     input:
         network=RESOURCES + "networks/elec_s{simpl}.nc",
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson",
diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index dc5fb7cc..34147af3 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -726,23 +726,23 @@ if __name__ == "__main__":
 
     costs = load_costs(
         snakemake.input.tech_costs,
-        snakemake.params["costs"],
-        snakemake.params["electricity"]["max_hours"],
+        snakemake.params.costs,
+        snakemake.params.electricity["max_hours"],
         Nyears,
     )
     ppl = load_powerplants(snakemake.input.powerplants)
 
-    if "renewable_carriers" in snakemake.params["electricity"]:
-        renewable_carriers = set(snakemake.params["electricity"]["renewable_carriers"])
+    if "renewable_carriers" in snakemake.params.electricity:
+        renewable_carriers = set(snakemake.params.electricity["renewable_carriers"])
     else:
         logger.warning(
             "Missing key `renewable_carriers` under config entry `electricity`. "
             "In future versions, this will raise an error. "
             "Falling back to carriers listed under `renewable`."
         )
-        renewable_carriers = snakemake.params["renewable"]
+        renewable_carriers = snakemake.params.renewable
 
-    extendable_carriers = snakemake.params["electricity"]["extendable_carriers"]
+    extendable_carriers = snakemake.params.electricity["extendable_carriers"]
     if not (set(renewable_carriers) & set(extendable_carriers["Generator"])):
         logger.warning(
             "No renewables found in config entry `extendable_carriers`. "
@@ -750,18 +750,18 @@ if __name__ == "__main__":
             "Falling back to all renewables."
         )
 
-    conventional_carriers = snakemake.params["electricity"]["conventional_carriers"]
+    conventional_carriers = snakemake.params.electricity["conventional_carriers"]
 
     attach_load(
         n,
         snakemake.input.regions,
         snakemake.input.load,
         snakemake.input.nuts3_shapes,
-        snakemake.params["countries"],
-        snakemake.params["scaling_factor"],
+        snakemake.params.countries,
+        snakemake.params.scaling_factor,
     )
 
-    update_transmission_costs(n, costs, snakemake.params["length_factor"])
+    update_transmission_costs(n, costs, snakemake.params.length_factor)
 
     conventional_inputs = {
         k: v for k, v in snakemake.input.items() if k.startswith("conventional_")
@@ -772,7 +772,7 @@ if __name__ == "__main__":
         ppl,
         conventional_carriers,
         extendable_carriers,
-        snakemake.params["conventional"],
+        snakemake.params.conventional,
         conventional_inputs,
     )
 
@@ -782,11 +782,11 @@ if __name__ == "__main__":
         snakemake.input,
         renewable_carriers,
         extendable_carriers,
-        snakemake.params["length_factor"],
+        snakemake.params.length_factor,
     )
 
     if "hydro" in renewable_carriers:
-        para = snakemake.params["renewable"]["hydro"]
+        para = snakemake.params.renewable["hydro"]
         attach_hydro(
             n,
             costs,
@@ -797,7 +797,7 @@ if __name__ == "__main__":
             **para,
         )
 
-    if "estimate_renewable_capacities" not in snakemake.params["electricity"]:
+    if "estimate_renewable_capacities" not in snakemake.params.electricity:
         logger.warning(
             "Missing key `estimate_renewable_capacities` under config entry `electricity`. "
             "In future versions, this will raise an error. "
@@ -805,18 +805,18 @@ if __name__ == "__main__":
         )
         if (
             "estimate_renewable_capacities_from_capacity_stats"
-            in snakemake.params["electricity"]
+            in snakemake.params.electricity
         ):
             estimate_renewable_caps = {
                 "enable": True,
-                **snakemake.params["electricity"][
+                **snakemake.params.electricity[
                     "estimate_renewable_capacities_from_capacity_stats"
                 ],
             }
         else:
             estimate_renewable_caps = {"enable": False}
     else:
-        estimate_renewable_caps = snakemake.params["electricity"][
+        estimate_renewable_caps = snakemake.params.electricity[
             "estimate_renewable_capacities"
         ]
     if "enable" not in estimate_renewable_caps:
@@ -832,18 +832,18 @@ if __name__ == "__main__":
             "Falling back to whether `renewable_capacities_from_opsd` is non-empty."
         )
         from_opsd = bool(
-            snakemake.params["electricity"].get("renewable_capacities_from_opsd", False)
+            snakemake.params.electricity.get("renewable_capacities_from_opsd", False)
         )
         estimate_renewable_caps["from_opsd"] = from_opsd
 
     if estimate_renewable_caps["enable"]:
         if estimate_renewable_caps["from_opsd"]:
-            tech_map = snakemake.params["electricity"]["estimate_renewable_capacities"][
+            tech_map = snakemake.params.electricity["estimate_renewable_capacities"][
                 "technology_mapping"
             ]
             attach_OPSD_renewables(n, tech_map)
         estimate_renewable_capacities(
-            n, snakemake.params["electricity"], snakemake.params["countries"]
+            n, snakemake.params.electricity, snakemake.params.countries
         )
 
     update_p_nom_max(n)
diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py
index 91463c23..9e90d42b 100644
--- a/scripts/build_renewable_profiles.py
+++ b/scripts/build_renewable_profiles.py
@@ -309,7 +309,7 @@ if __name__ == "__main__":
         p_nom_max = capacities / max_cap_factor
     else:
         raise AssertionError(
-            'params key `potential` should be one of "simple" '
+            'Config key `potential` should be one of "simple" '
             f'(default) or "conservative", not "{p_nom_max_meth}"'
         )
 
diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py
index db0fb2c2..52685af2 100644
--- a/scripts/cluster_network.py
+++ b/scripts/cluster_network.py
@@ -186,7 +186,7 @@ def get_feature_for_hac(n, buses_i=None, feature=None):
     if "offwind" in carriers:
         carriers.remove("offwind")
         carriers = np.append(
-            carriers, network.generators.carrier.filter(like="offwind").unique()
+            carriers, n.generators.carrier.filter(like="offwind").unique()
         )
 
     if feature.split("-")[1] == "cap":
@@ -463,26 +463,18 @@ if __name__ == "__main__":
         snakemake = mock_snakemake("cluster_network", simpl="", clusters="5")
     configure_logging(snakemake)
 
+    params = snakemake.params
+    solver_name = snakemake.config["solving"]["solver"]["name"]
+
     n = pypsa.Network(snakemake.input.network)
 
-    focus_weights = snakemake.config.get("focus_weights", None)
-
-    renewable_carriers = pd.Index(
-        [
-            tech
-            for tech in n.generators.carrier.unique()
-            if tech in snakemake.params["renewable"]
-        ]
-    )
-
-    exclude_carriers = snakemake.params["clustering"]["cluster_network"].get(
-        "exclude_carriers", []
-    )
+    exclude_carriers = params.cluster_network["exclude_carriers"]
     aggregate_carriers = set(n.generators.carrier) - set(exclude_carriers)
     if snakemake.wildcards.clusters.endswith("m"):
         n_clusters = int(snakemake.wildcards.clusters[:-1])
-        conventional = set(snakemake.params["conventional_carriers"])
-        aggregate_carriers = conventional.intersection(aggregate_carriers)
+        aggregate_carriers = set(params.conventional_carriers).intersection(
+            aggregate_carriers
+        )
     elif snakemake.wildcards.clusters == "all":
         n_clusters = len(n.buses)
     else:
@@ -496,13 +488,12 @@ if __name__ == "__main__":
             n, busmap, linemap, linemap, pd.Series(dtype="O")
         )
     else:
-        line_length_factor = snakemake.params["length_factor"]
         Nyears = n.snapshot_weightings.objective.sum() / 8760
 
         hvac_overhead_cost = load_costs(
             snakemake.input.tech_costs,
-            snakemake.params["costs"],
-            snakemake.params["max_hours"],
+            params.costs,
+            params.max_hours,
             Nyears,
         ).at["HVAC overhead", "capital_cost"]
 
@@ -513,16 +504,16 @@ if __name__ == "__main__":
             ).all() or x.isnull().all(), "The `potential` configuration option must agree for all renewable carriers, for now!"
             return v
 
-        aggregation_strategies = snakemake.params["clustering"].get(
-            "aggregation_strategies", {}
-        )
         # translate str entries of aggregation_strategies to pd.Series functions:
         aggregation_strategies = {
-            p: {k: getattr(pd.Series, v) for k, v in aggregation_strategies[p].items()}
-            for p in aggregation_strategies.keys()
+            p: {
+                k: getattr(pd.Series, v)
+                for k, v in params.aggregation_strategies[p].items()
+            }
+            for p in params.aggregation_strategies.keys()
         }
 
-        custom_busmap = snakemake.params["custom_busmap"]
+        custom_busmap = params.custom_busmap
         if custom_busmap:
             custom_busmap = pd.read_csv(
                 snakemake.input.custom_busmap, index_col=0, squeeze=True
@@ -530,21 +521,18 @@ if __name__ == "__main__":
             custom_busmap.index = custom_busmap.index.astype(str)
             logger.info(f"Imported custom busmap from {snakemake.input.custom_busmap}")
 
-        cluster_config = snakemake.config.get("clustering", {}).get(
-            "cluster_network", {}
-        )
         clustering = clustering_for_n_clusters(
             n,
             n_clusters,
             custom_busmap,
             aggregate_carriers,
-            line_length_factor,
-            aggregation_strategies,
-            snakemake.params["solver_name"],
-            cluster_config.get("algorithm", "hac"),
-            cluster_config.get("feature", "solar+onwind-time"),
+            params.length_factor,
+            params.aggregation_strategies,
+            solver_name,
+            params.cluster_network["algorithm"],
+            params.cluster_network["feature"],
             hvac_overhead_cost,
-            focus_weights,
+            params.focus_weights,
         )
 
     update_p_nom_max(clustering.network)
diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 83e932a3..79161760 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -149,17 +149,17 @@ def simplify_network_to_380(n):
     return n, trafo_map
 
 
-def _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param):
+def _prepare_connection_costs_per_link(n, costs, renewable_carriers, length_factor):
     if n.links.empty:
         return {}
 
     connection_costs_per_link = {}
 
-    for tech in renewable_param:
+    for tech in renewable_carriers:
         if tech.startswith("offwind"):
             connection_costs_per_link[tech] = (
                 n.links.length
-                * length_factor_param
+                * length_factor
                 * (
                     n.links.underwater_fraction
                     * costs.at[tech + "-connection-submarine", "capital_cost"]
@@ -175,14 +175,14 @@ def _compute_connection_costs_to_bus(
     n,
     busmap,
     costs,
-    renewable_param,
-    length_factor_param,
+    renewable_carriers,
+    length_factor,
     connection_costs_per_link=None,
     buses=None,
 ):
     if connection_costs_per_link is None:
         connection_costs_per_link = _prepare_connection_costs_per_link(
-            n, costs, renewable_param, length_factor_param
+            n, costs, renewable_carriers, length_factor
         )
 
     if buses is None:
@@ -276,10 +276,10 @@ def _aggregate_and_move_components(
 def simplify_links(
     n,
     costs,
-    renewable_param,
-    length_factor_param,
-    p_max_pu_param,
-    exclude_carriers_param,
+    renewables,
+    length_factor,
+    p_max_pu,
+    exclude_carriers,
     output,
     aggregation_strategies=dict(),
 ):
@@ -333,7 +333,7 @@ def simplify_links(
     busmap = n.buses.index.to_series()
 
     connection_costs_per_link = _prepare_connection_costs_per_link(
-        n, costs, renewable_param, length_factor_param
+        n, costs, renewables, length_factor
     )
     connection_costs_to_bus = pd.DataFrame(
         0.0, index=n.buses.index, columns=list(connection_costs_per_link)
@@ -355,8 +355,8 @@ def simplify_links(
                 n,
                 busmap,
                 costs,
-                renewable_param,
-                length_factor_param,
+                renewables,
+                length_factor,
                 connection_costs_per_link,
                 buses,
             )
@@ -378,8 +378,8 @@ def simplify_links(
                     / lengths.sum()
                     * n.links.loc[all_links, "underwater_fraction"]
                 ),
-                p_max_pu=p_max_pu_param,
-                p_min_pu=-p_max_pu_param,
+                p_max_pu=p_max_pu,
+                p_min_pu=-p_max_pu,
                 underground=False,
                 under_construction=False,
             )
@@ -407,7 +407,7 @@ def simplify_links(
         connection_costs_to_bus,
         output,
         aggregation_strategies=aggregation_strategies,
-        exclude_carriers=exclude_carriers_param,
+        exclude_carriers=exclude_carriers,
     )
     return n, busmap
 
@@ -415,34 +415,29 @@ def simplify_links(
 def remove_stubs(
     n,
     costs,
-    renewable_param,
-    length_factor_param,
-    clustering_param,
-    exclude_carriers_param,
+    renewable_carriers,
+    length_factor,
+    simplify_network,
     output,
     aggregation_strategies=dict(),
 ):
     logger.info("Removing stubs")
 
-    across_borders = clustering_param["simplify_network"].get(
-        "remove_stubs_across_borders", True
-    )
+    across_borders = simplify_network["remove_stubs_across_borders"]
     matching_attrs = [] if across_borders else ["country"]
     busmap = busmap_by_stubs(n, matching_attrs)
 
     connection_costs_to_bus = _compute_connection_costs_to_bus(
-        n, busmap, costs, renewable_param, length_factor_param
+        n, busmap, costs, renewable_carriers, length_factor
     )
 
-    exclude_carriers = clustering_param["simplify_network"].get("exclude_carriers", [])
-
     _aggregate_and_move_components(
         n,
         busmap,
         connection_costs_to_bus,
         output,
         aggregation_strategies=aggregation_strategies,
-        exclude_carriers=exclude_carriers,
+        exclude_carriers=simplify_network["exclude_carriers"],
     )
 
     return n, busmap
@@ -504,32 +499,23 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None):
 def cluster(
     n,
     n_clusters,
-    focus_weights_param,
-    renewable_param,
-    solver_name_param,
+    focus_weights,
+    solver_name,
     algorithm="hac",
     feature=None,
     aggregation_strategies=dict(),
 ):
     logger.info(f"Clustering to {n_clusters} buses")
 
-    renewable_carriers = pd.Index(
-        [
-            tech
-            for tech in n.generators.carrier.unique()
-            if tech.split("-", 2)[0] in renewable_param
-        ]
-    )
-
     clustering = clustering_for_n_clusters(
         n,
         n_clusters,
         custom_busmap=False,
         aggregation_strategies=aggregation_strategies,
-        solver_name=solver_name_param,
+        solver_name=solver_name,
         algorithm=algorithm,
         feature=feature,
-        focus_weights=focus_weights_param,
+        focus_weights=focus_weights,
     )
 
     return clustering.network, clustering.busmap
@@ -542,77 +528,69 @@ if __name__ == "__main__":
         snakemake = mock_snakemake("simplify_network", simpl="")
     configure_logging(snakemake)
 
-    n = pypsa.Network(snakemake.input.network)
+    params = snakemake.params
+    solver_name = snakemake.config["solving"]["solver"]["name"]
+
+    n = pypsa.Network(snakemake.input.network)
+    Nyears = n.snapshot_weightings.objective.sum() / 8760
 
-    aggregation_strategies = snakemake.params["clustering"].get(
-        "aggregation_strategies", {}
-    )
     # translate str entries of aggregation_strategies to pd.Series functions:
     aggregation_strategies = {
-        p: {k: getattr(pd.Series, v) for k, v in aggregation_strategies[p].items()}
-        for p in aggregation_strategies.keys()
+        p: {
+            k: getattr(pd.Series, v)
+            for k, v in params.aggregation_strategies[p].items()
+        }
+        for p in params.aggregation_strategies.keys()
     }
 
     n, trafo_map = simplify_network_to_380(n)
 
-    Nyears = n.snapshot_weightings.objective.sum() / 8760
-
     technology_costs = load_costs(
         snakemake.input.tech_costs,
-        snakemake.params["costs"],
-        snakemake.params["max_hours"],
+        params.costs,
+        params.max_hours,
         Nyears,
     )
 
     n, simplify_links_map = simplify_links(
         n,
         technology_costs,
-        snakemake.params["renewable"],
-        snakemake.params["length_factor"],
-        snakemake.params["p_max_pu"],
-        snakemake.params["exclude_carriers"],
+        params.renewable_carriers,
+        params.length_factor,
+        params.p_max_pu,
+        params.simplify_network["exclude_carriers"],
         snakemake.output,
         aggregation_strategies,
     )
 
     busmaps = [trafo_map, simplify_links_map]
 
-    cluster_param = snakemake.params["clustering"]["simplify_network"]
-    if cluster_param.get("remove_stubs", True):
+    if params.simplify_network["remove_stubs"]:
         n, stub_map = remove_stubs(
             n,
             technology_costs,
-            snakemake.params["renewable"],
-            snakemake.params["length_factor"],
-            snakemake.params["clustering"],
-            snakemake.params["exclude_carriers"],
+            params.renewable_carriers,
+            params.length_factor,
+            params.simplify_network,
             snakemake.output,
             aggregation_strategies=aggregation_strategies,
         )
         busmaps.append(stub_map)
 
-    if cluster_param.get("to_substations", False):
+    if params.simplify_network["to_substations"]:
         n, substation_map = aggregate_to_substations(n, aggregation_strategies)
         busmaps.append(substation_map)
 
     # treatment of outliers (nodes without a profile for considered carrier):
     # all nodes that have no profile of the given carrier are being aggregated to closest neighbor
-    if (
-        snakemake.config.get("clustering", {})
-        .get("cluster_network", {})
-        .get("algorithm", "hac")
-        == "hac"
-        or cluster_param.get("algorithm", "hac") == "hac"
-    ):
-        carriers = (
-            cluster_param.get("feature", "solar+onwind-time").split("-")[0].split("+")
-        )
+    if params.simplify_network["algorithm"] == "hac":
+        carriers = params.simplify_network["feature"].split("-")[0].split("+")
         for carrier in carriers:
             buses_i = list(
                 set(n.buses.index) - set(n.generators.query("carrier == @carrier").bus)
             )
             logger.info(
-                f"clustering preparaton (hac): aggregating {len(buses_i)} buses of type {carrier}."
+                f"clustering preparation (hac): aggregating {len(buses_i)} buses of type {carrier}."
             )
             n, busmap_hac = aggregate_to_substations(n, aggregation_strategies, buses_i)
             busmaps.append(busmap_hac)
@@ -621,11 +599,10 @@ if __name__ == "__main__":
         n, cluster_map = cluster(
             n,
             int(snakemake.wildcards.simpl),
-            snakemake.params["focus_weights"],
-            snakemake.params["renewable"],
-            snakemake.params["solver_name"],
-            cluster_param.get("algorithm", "hac"),
-            cluster_param.get("feature", None),
+            params.focus_weights,
+            solver_name,
+            params.simplify_network["algorithm"],
+            params.simplify_network["feature"],
             aggregation_strategies,
         )
         busmaps.append(cluster_map)

From 1d10073514a66f09e6821364f03a57f99cff2352 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Thu, 15 Jun 2023 18:52:25 +0200
Subject: [PATCH 127/176] review params implementation; reproduce objective
 values in CI

---
 config/test/config.myopic.yaml                |  8 ++
 config/test/config.overnight.yaml             |  8 ++
 rules/build_electricity.smk                   |  2 +-
 rules/build_sector.smk                        |  2 +-
 scripts/add_brownfield.py                     |  6 +-
 scripts/add_electricity.py                    | 93 +++++--------------
 scripts/add_existing_baseyear.py              | 20 ++--
 scripts/add_extra_components.py               | 26 +++---
 scripts/build_ammonia_production.py           |  2 +-
 scripts/build_biomass_potentials.py           |  2 +-
 scripts/build_bus_regions.py                  |  2 +-
 scripts/build_cop_profiles.py                 |  2 +-
 scripts/build_cutout.py                       |  4 +-
 scripts/build_electricity_demand.py           | 12 +--
 scripts/build_energy_totals.py                |  8 +-
 scripts/build_heat_demand.py                  |  2 +-
 scripts/build_hydro_profile.py                |  4 +-
 scripts/build_industrial_distribution_key.py  |  4 +-
 ...ustrial_energy_demand_per_country_today.py |  4 +-
 ...build_industrial_production_per_country.py |  6 +-
 ...ustrial_production_per_country_tomorrow.py |  2 +-
 scripts/build_industry_sector_ratios.py       |  4 +-
 scripts/build_powerplants.py                  |  6 +-
 scripts/build_renewable_profiles.py           |  2 +-
 scripts/build_retro_cost.py                   |  4 +-
 scripts/build_sequestration_potentials.py     |  2 +-
 scripts/build_shapes.py                       |  6 +-
 scripts/build_solar_thermal_profiles.py       |  4 +-
 scripts/build_temperature_profiles.py         |  2 +-
 scripts/build_transport_demand.py             |  4 +-
 scripts/make_summary.py                       | 20 ++--
 scripts/plot_network.py                       | 16 ++--
 scripts/plot_summary.py                       | 24 ++---
 scripts/prepare_network.py                    | 20 ++--
 scripts/prepare_sector_network.py             | 40 ++++----
 scripts/retrieve_databundle.py                |  2 +-
 scripts/solve_network.py                      | 42 ++++-----
 scripts/solve_operations_network.py           |  2 +-
 38 files changed, 188 insertions(+), 231 deletions(-)

diff --git a/config/test/config.myopic.yaml b/config/test/config.myopic.yaml
index efa03136..0bb85ec6 100644
--- a/config/test/config.myopic.yaml
+++ b/config/test/config.myopic.yaml
@@ -31,6 +31,14 @@ snapshots:
   end: "2013-03-08"
 
 electricity:
+  co2limit: 100.e+6
+
+  extendable_carriers:
+    Generator: [OCGT]
+    StorageUnit: [battery]
+    Store: [H2]
+    Link: [H2 pipeline]
+
   renewable_carriers: [solar, onwind, offwind-ac, offwind-dc]
 
 atlite:
diff --git a/config/test/config.overnight.yaml b/config/test/config.overnight.yaml
index fb468ded..a2a0f5a4 100644
--- a/config/test/config.overnight.yaml
+++ b/config/test/config.overnight.yaml
@@ -28,6 +28,14 @@ snapshots:
   end: "2013-03-08"
 
 electricity:
+  co2limit: 100.e+6
+
+  extendable_carriers:
+    Generator: [OCGT]
+    StorageUnit: [battery]
+    Store: [H2]
+    Link: [H2 pipeline]
+
   renewable_carriers: [solar, onwind, offwind-ac, offwind-dc]
 
 atlite:
diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index fd66cc04..9d4315df 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -394,7 +394,7 @@ rule cluster_network:
 rule add_extra_components:
     params:
         costs=config["costs"],
-        ext_carriers=config["electricity"]["extendable_carriers"],
+        extendable_carriers=config["electricity"]["extendable_carriers"],
         max_hours=config["electricity"]["max_hours"],
     input:
         network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc",
diff --git a/rules/build_sector.smk b/rules/build_sector.smk
index 3714cbad..e15394ff 100644
--- a/rules/build_sector.smk
+++ b/rules/build_sector.smk
@@ -406,7 +406,7 @@ rule build_ammonia_production:
 rule build_industry_sector_ratios:
     params:
         industry=config["industry"],
-        sector_amonia=config["sector"].get("ammonia", False),
+        ammonia=config["sector"].get("ammonia", False),
     input:
         ammonia_production=RESOURCES + "ammonia_production.csv",
         idees="data/jrc-idees-2015",
diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py
index 48dffbb9..b33d2f19 100644
--- a/scripts/add_brownfield.py
+++ b/scripts/add_brownfield.py
@@ -49,7 +49,7 @@ def add_brownfield(n, n_p, year):
             )
         ]
 
-        threshold = snakemake.params["threshold_capacity"]
+        threshold = snakemake.params.threshold_capacity
 
         if not chp_heat.empty:
             threshold_chp_heat = (
@@ -87,7 +87,7 @@ def add_brownfield(n, n_p, year):
 
         # deal with gas network
         pipe_carrier = ["gas pipeline"]
-        if snakemake.params["H2_retrofit"]:
+        if snakemake.params.H2_retrofit:
             # drop capacities of previous year to avoid duplicating
             to_drop = n.links.carrier.isin(pipe_carrier) & (n.links.build_year != year)
             n.mremove("Link", n.links.loc[to_drop].index)
@@ -98,7 +98,7 @@ def add_brownfield(n, n_p, year):
                 & (n.links.build_year != year)
             ].index
             gas_pipes_i = n.links[n.links.carrier.isin(pipe_carrier)].index
-            CH4_per_H2 = 1 / snakemake.params["H2_retrofit_capacity_per_CH4"]
+            CH4_per_H2 = 1 / snakemake.params.H2_retrofit_capacity_per_CH4
             fr = "H2 pipeline retrofitted"
             to = "gas pipeline"
             # today's pipe capacity
diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 34147af3..d98dc767 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -652,13 +652,7 @@ def attach_OPSD_renewables(n, tech_map):
         n.generators.p_nom_min.update(gens.bus.map(caps).dropna())
 
 
-def estimate_renewable_capacities(n, electricity_params, countries):
-    year = electricity_params["estimate_renewable_capacities"]["year"]
-    tech_map = electricity_params["estimate_renewable_capacities"]["technology_mapping"]
-    expansion_limit = electricity_params["estimate_renewable_capacities"][
-        "expansion_limit"
-    ]
-
+def estimate_renewable_capacities(n, year, tech_map, expansion_limit, countries):
     if not len(countries) or not len(tech_map):
         return
 
@@ -721,48 +715,42 @@ if __name__ == "__main__":
         snakemake = mock_snakemake("add_electricity")
     configure_logging(snakemake)
 
+    params = snakemake.params
+
     n = pypsa.Network(snakemake.input.base_network)
     Nyears = n.snapshot_weightings.objective.sum() / 8760.0
 
     costs = load_costs(
         snakemake.input.tech_costs,
-        snakemake.params.costs,
-        snakemake.params.electricity["max_hours"],
+        params.costs,
+        params.electricity["max_hours"],
         Nyears,
     )
     ppl = load_powerplants(snakemake.input.powerplants)
 
-    if "renewable_carriers" in snakemake.params.electricity:
-        renewable_carriers = set(snakemake.params.electricity["renewable_carriers"])
+    if "renewable_carriers" in params.electricity:
+        renewable_carriers = set(params.electricity["renewable_carriers"])
     else:
         logger.warning(
             "Missing key `renewable_carriers` under config entry `electricity`. "
             "In future versions, this will raise an error. "
             "Falling back to carriers listed under `renewable`."
         )
-        renewable_carriers = snakemake.params.renewable
-
-    extendable_carriers = snakemake.params.electricity["extendable_carriers"]
-    if not (set(renewable_carriers) & set(extendable_carriers["Generator"])):
-        logger.warning(
-            "No renewables found in config entry `extendable_carriers`. "
-            "In future versions, these have to be explicitly listed. "
-            "Falling back to all renewables."
-        )
-
-    conventional_carriers = snakemake.params.electricity["conventional_carriers"]
+        renewable_carriers = params.renewable
 
     attach_load(
         n,
         snakemake.input.regions,
         snakemake.input.load,
         snakemake.input.nuts3_shapes,
-        snakemake.params.countries,
-        snakemake.params.scaling_factor,
+        params.countries,
+        params.scaling_factor,
     )
 
-    update_transmission_costs(n, costs, snakemake.params.length_factor)
+    update_transmission_costs(n, costs, params.length_factor)
 
+    extendable_carriers = params.electricity["extendable_carriers"]
+    conventional_carriers = params.electricity["conventional_carriers"]
     conventional_inputs = {
         k: v for k, v in snakemake.input.items() if k.startswith("conventional_")
     }
@@ -772,7 +760,7 @@ if __name__ == "__main__":
         ppl,
         conventional_carriers,
         extendable_carriers,
-        snakemake.params.conventional,
+        params.conventional,
         conventional_inputs,
     )
 
@@ -782,11 +770,11 @@ if __name__ == "__main__":
         snakemake.input,
         renewable_carriers,
         extendable_carriers,
-        snakemake.params.length_factor,
+        params.length_factor,
     )
 
     if "hydro" in renewable_carriers:
-        para = snakemake.params.renewable["hydro"]
+        para = params.renewable["hydro"]
         attach_hydro(
             n,
             costs,
@@ -797,53 +785,16 @@ if __name__ == "__main__":
             **para,
         )
 
-    if "estimate_renewable_capacities" not in snakemake.params.electricity:
-        logger.warning(
-            "Missing key `estimate_renewable_capacities` under config entry `electricity`. "
-            "In future versions, this will raise an error. "
-            "Falling back to whether ``estimate_renewable_capacities_from_capacity_stats`` is in the config."
-        )
-        if (
-            "estimate_renewable_capacities_from_capacity_stats"
-            in snakemake.params.electricity
-        ):
-            estimate_renewable_caps = {
-                "enable": True,
-                **snakemake.params.electricity[
-                    "estimate_renewable_capacities_from_capacity_stats"
-                ],
-            }
-        else:
-            estimate_renewable_caps = {"enable": False}
-    else:
-        estimate_renewable_caps = snakemake.params.electricity[
-            "estimate_renewable_capacities"
-        ]
-    if "enable" not in estimate_renewable_caps:
-        logger.warning(
-            "Missing key `enable` under config entry `estimate_renewable_capacities`. "
-            "In future versions, this will raise an error. Falling back to False."
-        )
-        estimate_renewable_caps = {"enable": False}
-    if "from_opsd" not in estimate_renewable_caps:
-        logger.warning(
-            "Missing key `from_opsd` under config entry `estimate_renewable_capacities`. "
-            "In future versions, this will raise an error. "
-            "Falling back to whether `renewable_capacities_from_opsd` is non-empty."
-        )
-        from_opsd = bool(
-            snakemake.params.electricity.get("renewable_capacities_from_opsd", False)
-        )
-        estimate_renewable_caps["from_opsd"] = from_opsd
-
+    estimate_renewable_caps = params.electricity["estimate_renewable_capacities"]
     if estimate_renewable_caps["enable"]:
+        tech_map = estimate_renewable_caps["technology_mapping"]
+        expansion_limit = estimate_renewable_caps["expansion_limit"]
+        year = estimate_renewable_caps["year"]
+
         if estimate_renewable_caps["from_opsd"]:
-            tech_map = snakemake.params.electricity["estimate_renewable_capacities"][
-                "technology_mapping"
-            ]
             attach_OPSD_renewables(n, tech_map)
         estimate_renewable_capacities(
-            n, snakemake.params.electricity, snakemake.params.countries
+            n, year, tech_map, expansion_limit, params.countries
         )
 
     update_p_nom_max(n)
diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py
index bc1d3a05..52cb585e 100644
--- a/scripts/add_existing_baseyear.py
+++ b/scripts/add_existing_baseyear.py
@@ -157,7 +157,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas
     # Fill missing DateOut
     dateout = (
         df_agg.loc[biomass_i, "DateIn"]
-        + snakemake.params["costs"]["fill_values"]["lifetime"]
+        + snakemake.params.costs["fill_values"]["lifetime"]
     )
     df_agg.loc[biomass_i, "DateOut"] = df_agg.loc[biomass_i, "DateOut"].fillna(dateout)
 
@@ -218,7 +218,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas
         capacity = df.loc[grouping_year, generator]
         capacity = capacity[~capacity.isna()]
         capacity = capacity[
-            capacity > snakemake.params["existing_capacities"]["threshold_capacity"]
+            capacity > snakemake.params.existing_capacities["threshold_capacity"]
         ]
         suffix = "-ac" if generator == "offwind" else ""
         name_suffix = f" {generator}{suffix}-{grouping_year}"
@@ -582,7 +582,7 @@ def add_heating_capacities_installed_before_baseyear(
             )
 
             # delete links with capacities below threshold
-            threshold = snakemake.params["existing_capacities"]["threshold_capacity"]
+            threshold = snakemake.params.existing_capacities["threshold_capacity"]
             n.mremove(
                 "Link",
                 [
@@ -612,10 +612,10 @@ if __name__ == "__main__":
 
     update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts)
 
-    options = snakemake.params["sector"]
+    options = snakemake.params.sector
     opts = snakemake.wildcards.sector_opts.split("-")
 
-    baseyear = snakemake.params["baseyear"]
+    baseyear = snakemake.params.baseyear
 
     overrides = override_component_attrs(snakemake.input.overrides)
     n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
@@ -626,14 +626,12 @@ if __name__ == "__main__":
     Nyears = n.snapshot_weightings.generators.sum() / 8760.0
     costs = prepare_costs(
         snakemake.input.costs,
-        snakemake.params["costs"],
+        snakemake.params.costs,
         Nyears,
     )
 
-    grouping_years_power = snakemake.params["existing_capacities"][
-        "grouping_years_power"
-    ]
-    grouping_years_heat = snakemake.params["existing_capacities"]["grouping_years_heat"]
+    grouping_years_power = snakemake.params.existing_capacities["grouping_years_power"]
+    grouping_years_heat = snakemake.params.existing_capacities["grouping_years_heat"]
     add_power_capacities_installed_before_baseyear(
         n, grouping_years_power, costs, baseyear
     )
@@ -650,7 +648,7 @@ if __name__ == "__main__":
             .to_pandas()
             .reindex(index=n.snapshots)
         )
-        default_lifetime = snakemake.params["costs"]["fill_values"]["lifetime"]
+        default_lifetime = snakemake.params.costs["fill_values"]["lifetime"]
         add_heating_capacities_installed_before_baseyear(
             n,
             baseyear,
diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py
index ce4e533e..f034a7f2 100644
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@@ -67,8 +67,8 @@ idx = pd.IndexSlice
 logger = logging.getLogger(__name__)
 
 
-def attach_storageunits(n, costs, ext_carriers, max_hours):
-    carriers = ext_carriers["StorageUnit"]
+def attach_storageunits(n, costs, extendable_carriers, max_hours):
+    carriers = extendable_carriers["StorageUnit"]
 
     _add_missing_carriers_from_costs(n, costs, carriers)
 
@@ -98,8 +98,8 @@ def attach_storageunits(n, costs, ext_carriers, max_hours):
         )
 
 
-def attach_stores(n, costs, ext_carriers):
-    carriers = ext_carriers["Store"]
+def attach_stores(n, costs, extendable_carriers):
+    carriers = extendable_carriers["Store"]
 
     _add_missing_carriers_from_costs(n, costs, carriers)
 
@@ -186,10 +186,10 @@ def attach_stores(n, costs, ext_carriers):
         )
 
 
-def attach_hydrogen_pipelines(n, costs, ext_carriers):
-    as_stores = ext_carriers.get("Store", [])
+def attach_hydrogen_pipelines(n, costs, extendable_carriers):
+    as_stores = extendable_carriers.get("Store", [])
 
-    if "H2 pipeline" not in ext_carriers.get("Link", []):
+    if "H2 pipeline" not in extendable_carriers.get("Link", []):
         return
 
     assert "H2" in as_stores, (
@@ -233,17 +233,17 @@ if __name__ == "__main__":
     configure_logging(snakemake)
 
     n = pypsa.Network(snakemake.input.network)
-    ext_carriers = snakemake.params["ext_carriers"]
-    max_hours = snakemake.params["max_hours"]
+    extendable_carriers = snakemake.params.extendable_carriers
+    max_hours = snakemake.params.max_hours
 
     Nyears = n.snapshot_weightings.objective.sum() / 8760.0
     costs = load_costs(
-        snakemake.input.tech_costs, snakemake.params["costs"], max_hours, Nyears
+        snakemake.input.tech_costs, snakemake.params.costs, max_hours, Nyears
     )
 
-    attach_storageunits(n, costs, ext_carriers, max_hours)
-    attach_stores(n, costs, ext_carriers)
-    attach_hydrogen_pipelines(n, costs, ext_carriers)
+    attach_storageunits(n, costs, extendable_carriers, max_hours)
+    attach_stores(n, costs, extendable_carriers)
+    attach_hydrogen_pipelines(n, costs, extendable_carriers)
 
     add_nice_carrier_names(n, snakemake.config)
 
diff --git a/scripts/build_ammonia_production.py b/scripts/build_ammonia_production.py
index 6f03324f..1bcdf9ae 100644
--- a/scripts/build_ammonia_production.py
+++ b/scripts/build_ammonia_production.py
@@ -30,7 +30,7 @@ if __name__ == "__main__":
     ammonia.index = cc.convert(ammonia.index, to="iso2")
 
     years = [str(i) for i in range(2013, 2018)]
-    countries = ammonia.index.intersection(snakemake.params["countries"])
+    countries = ammonia.index.intersection(snakemake.params.countries)
     ammonia = ammonia.loc[countries, years].astype(float)
 
     # convert from ktonN to ktonNH3
diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py
index 0b423ad5..d200a78e 100644
--- a/scripts/build_biomass_potentials.py
+++ b/scripts/build_biomass_potentials.py
@@ -210,7 +210,7 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("build_biomass_potentials", simpl="", clusters="5")
 
-    params = snakemake.params["biomass"]
+    params = snakemake.params.biomass
     year = params["year"]
     scenario = params["scenario"]
 
diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py
index e9378792..a6500bb0 100644
--- a/scripts/build_bus_regions.py
+++ b/scripts/build_bus_regions.py
@@ -116,7 +116,7 @@ if __name__ == "__main__":
         snakemake = mock_snakemake("build_bus_regions")
     configure_logging(snakemake)
 
-    countries = snakemake.params["countries"]
+    countries = snakemake.params.countries
 
     n = pypsa.Network(snakemake.input.base_network)
 
diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py
index 983eda2d..4b1d952e 100644
--- a/scripts/build_cop_profiles.py
+++ b/scripts/build_cop_profiles.py
@@ -39,7 +39,7 @@ if __name__ == "__main__":
         for source in ["air", "soil"]:
             source_T = xr.open_dataarray(snakemake.input[f"temp_{source}_{area}"])
 
-            delta_T = snakemake.params["heat_pump_sink_T"] - source_T
+            delta_T = snakemake.params.heat_pump_sink_T - source_T
 
             cop = coefficient_of_performance(delta_T, source)
 
diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py
index 798588d9..9a7f9e00 100644
--- a/scripts/build_cutout.py
+++ b/scripts/build_cutout.py
@@ -106,9 +106,9 @@ if __name__ == "__main__":
         snakemake = mock_snakemake("build_cutout", cutout="europe-2013-era5")
     configure_logging(snakemake)
 
-    cutout_params = snakemake.params["cutouts"][snakemake.wildcards.cutout]
+    cutout_params = snakemake.params.cutouts[snakemake.wildcards.cutout]
 
-    snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"])
+    snapshots = pd.date_range(freq="h", **snakemake.params.snapshots)
     time = [snapshots[0], snapshots[-1]]
     cutout_params["time"] = slice(*cutout_params.get("time", time))
 
diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py
index 4ef56d1d..ba1fb881 100755
--- a/scripts/build_electricity_demand.py
+++ b/scripts/build_electricity_demand.py
@@ -279,16 +279,16 @@ if __name__ == "__main__":
 
     configure_logging(snakemake)
 
-    powerstatistics = snakemake.params["load"]["power_statistics"]
-    interpolate_limit = snakemake.params["load"]["interpolate_limit"]
-    countries = snakemake.params["countries"]
-    snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"])
+    powerstatistics = snakemake.params.load["power_statistics"]
+    interpolate_limit = snakemake.params.load["interpolate_limit"]
+    countries = snakemake.params.countries
+    snapshots = pd.date_range(freq="h", **snakemake.params.snapshots)
     years = slice(snapshots[0], snapshots[-1])
-    time_shift = snakemake.params["load"]["time_shift_for_large_gaps"]
+    time_shift = snakemake.params.load["time_shift_for_large_gaps"]
 
     load = load_timeseries(snakemake.input[0], years, countries, powerstatistics)
 
-    if snakemake.params["load"]["manual_adjustments"]:
+    if snakemake.params.load["manual_adjustments"]:
         load = manual_adjustment(load, snakemake.input[0], powerstatistics)
 
     logger.info(f"Linearly interpolate gaps of size {interpolate_limit} and less.")
diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py
index 3e3cb485..891c4e2a 100644
--- a/scripts/build_energy_totals.py
+++ b/scripts/build_energy_totals.py
@@ -737,16 +737,16 @@ if __name__ == "__main__":
 
     logging.basicConfig(level=snakemake.config["logging"]["level"])
 
-    params = snakemake.params["energy"]
+    params = snakemake.params.energy
 
     nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index("index")
     population = nuts3["pop"].groupby(nuts3.country).sum()
 
-    countries = snakemake.params["countries"]
+    countries = snakemake.params.countries
     idees_countries = pd.Index(countries).intersection(eu28)
 
     data_year = params["energy_totals_year"]
-    report_year = snakemake.params["energy"]["eurostat_report_year"]
+    report_year = snakemake.params.energy["eurostat_report_year"]
     input_eurostat = snakemake.input.eurostat
     eurostat = build_eurostat(input_eurostat, countries, report_year, data_year)
     swiss = build_swiss(data_year)
@@ -756,7 +756,7 @@ if __name__ == "__main__":
     energy.to_csv(snakemake.output.energy_name)
 
     base_year_emissions = params["base_emissions_year"]
-    emissions_scope = snakemake.params["energy"]["emissions"]
+    emissions_scope = snakemake.params.energy["emissions"]
     eea_co2 = build_eea_co2(snakemake.input.co2, base_year_emissions, emissions_scope)
     eurostat_co2 = build_eurostat_co2(
         input_eurostat, countries, report_year, base_year_emissions
diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py
index 655df28f..73494260 100644
--- a/scripts/build_heat_demand.py
+++ b/scripts/build_heat_demand.py
@@ -27,7 +27,7 @@ if __name__ == "__main__":
     cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1)
     client = Client(cluster, asynchronous=True)
 
-    time = pd.date_range(freq="h", **snakemake.params["snapshots"])
+    time = pd.date_range(freq="h", **snakemake.params.snapshots)
     cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time)
 
     clustered_regions = (
diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py
index 26bf31c6..bed666f2 100644
--- a/scripts/build_hydro_profile.py
+++ b/scripts/build_hydro_profile.py
@@ -130,10 +130,10 @@ if __name__ == "__main__":
         snakemake = mock_snakemake("build_hydro_profile")
     configure_logging(snakemake)
 
-    params_hydro = snakemake.params["hydro"]
+    params_hydro = snakemake.params.hydro
     cutout = atlite.Cutout(snakemake.input.cutout)
 
-    countries = snakemake.params["countries"]
+    countries = snakemake.params.countries
     country_shapes = (
         gpd.read_file(snakemake.input.country_shapes)
         .set_index("name")["geometry"]
diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py
index a51ff8b2..e93e43c2 100644
--- a/scripts/build_industrial_distribution_key.py
+++ b/scripts/build_industrial_distribution_key.py
@@ -73,7 +73,7 @@ def prepare_hotmaps_database(regions):
 
     df[["srid", "coordinates"]] = df.geom.str.split(";", expand=True)
 
-    if snakemake.params["hotmaps_locate_missing"]:
+    if snakemake.params.hotmaps_locate_missing:
         df = locate_missing_industrial_sites(df)
 
     # remove those sites without valid locations
@@ -143,7 +143,7 @@ if __name__ == "__main__":
 
     logging.basicConfig(level=snakemake.config["logging"]["level"])
 
-    countries = snakemake.params["countries"]
+    countries = snakemake.params.countries
 
     regions = gpd.read_file(snakemake.input.regions_onshore).set_index("name")
 
diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py
index 9f8c47d0..9ca0d003 100644
--- a/scripts/build_industrial_energy_demand_per_country_today.py
+++ b/scripts/build_industrial_energy_demand_per_country_today.py
@@ -178,9 +178,9 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("build_industrial_energy_demand_per_country_today")
 
-    params = snakemake.params["industry"]
+    params = snakemake.params.industry
     year = params.get("reference_year", 2015)
-    countries = pd.Index(snakemake.params["countries"])
+    countries = pd.Index(snakemake.params.countries)
 
     demand = industrial_energy_demand(countries.intersection(eu28), year)
 
diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py
index 889c9ecd..74cb1949 100644
--- a/scripts/build_industrial_production_per_country.py
+++ b/scripts/build_industrial_production_per_country.py
@@ -279,11 +279,11 @@ if __name__ == "__main__":
 
     logging.basicConfig(level=snakemake.config["logging"]["level"])
 
-    countries = snakemake.params["countries"]
+    countries = snakemake.params.countries
 
-    year = snakemake.params["industry"]["reference_year"]
+    year = snakemake.params.industry["reference_year"]
 
-    params = snakemake.params["industry"]
+    params = snakemake.params.industry
 
     jrc_dir = snakemake.input.jrc
     eurostat_dir = snakemake.input.eurostat
diff --git a/scripts/build_industrial_production_per_country_tomorrow.py b/scripts/build_industrial_production_per_country_tomorrow.py
index 609170aa..ffed5195 100644
--- a/scripts/build_industrial_production_per_country_tomorrow.py
+++ b/scripts/build_industrial_production_per_country_tomorrow.py
@@ -15,7 +15,7 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake("build_industrial_production_per_country_tomorrow")
 
-    params = snakemake.params["industry"]
+    params = snakemake.params.industry
 
     investment_year = int(snakemake.wildcards.planning_horizons)
 
diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py
index fa9e5c18..45705002 100644
--- a/scripts/build_industry_sector_ratios.py
+++ b/scripts/build_industry_sector_ratios.py
@@ -439,7 +439,7 @@ def chemicals_industry():
 
     sector = "Ammonia"
     df[sector] = 0.0
-    if snakemake.params["sector_amonia"]:
+    if snakemake.params.ammonia:
         df.loc["ammonia", sector] = params["MWh_NH3_per_tNH3"]
     else:
         df.loc["hydrogen", sector] = params["MWh_H2_per_tNH3_electrolysis"]
@@ -1468,7 +1468,7 @@ if __name__ == "__main__":
     # TODO make params option
     year = 2015
 
-    params = snakemake.params["industry"]
+    params = snakemake.params.industry
 
     df = pd.concat(
         [
diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py
index 6edd4ac4..7f396e1e 100755
--- a/scripts/build_powerplants.py
+++ b/scripts/build_powerplants.py
@@ -115,7 +115,7 @@ if __name__ == "__main__":
     configure_logging(snakemake)
 
     n = pypsa.Network(snakemake.input.base_network)
-    countries = snakemake.params["countries"]
+    countries = snakemake.params.countries
 
     ppl = (
         pm.powerplants(from_url=True)
@@ -134,12 +134,12 @@ if __name__ == "__main__":
     ppl = ppl.query('not (Country in @available_countries and Fueltype == "Bioenergy")')
     ppl = pd.concat([ppl, opsd])
 
-    ppl_query = snakemake.params["powerplants_filter"]
+    ppl_query = snakemake.params.powerplants_filter
     if isinstance(ppl_query, str):
         ppl.query(ppl_query, inplace=True)
 
     # add carriers from own powerplant files:
-    custom_ppl_query = snakemake.params["custom_powerplants"]
+    custom_ppl_query = snakemake.params.custom_powerplants
     ppl = add_custom_powerplants(
         ppl, snakemake.input.custom_powerplants, custom_ppl_query
     )
diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py
index 9e90d42b..9f1f1a51 100644
--- a/scripts/build_renewable_profiles.py
+++ b/scripts/build_renewable_profiles.py
@@ -204,7 +204,7 @@ if __name__ == "__main__":
 
     nprocesses = int(snakemake.threads)
     noprogress = snakemake.config["run"].get("disable_progressbar", True)
-    params = snakemake.params["renewable"][snakemake.wildcards.technology]
+    params = snakemake.params.renewable[snakemake.wildcards.technology]
     resource = params["resource"]  # pv panel params / wind turbine params
     correction_factor = params.get("correction_factor", 1.0)
     capacity_per_sqkm = params["capacity_per_sqkm"]
diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py
index ac7eb4ae..c830415e 100644
--- a/scripts/build_retro_cost.py
+++ b/scripts/build_retro_cost.py
@@ -305,7 +305,7 @@ def prepare_building_stock_data():
     u_values.set_index(["country_code", "subsector", "bage", "type"], inplace=True)
 
     #  only take in config.yaml specified countries into account
-    countries = snakemake.params["countries"]
+    countries = snakemake.params.countries
     area_tot = area_tot.loc[countries]
 
     return u_values, country_iso_dic, countries, area_tot, area
@@ -1040,7 +1040,7 @@ if __name__ == "__main__":
 
     #  ********  config  *********************************************************
 
-    retro_opts = snakemake.params["retrofitting"]
+    retro_opts = snakemake.params.retrofitting
     interest_rate = retro_opts["interest_rate"]
     annualise_cost = retro_opts["annualise_cost"]  # annualise the investment costs
     tax_weighting = retro_opts[
diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py
index 9d26b0b9..0e59e55b 100644
--- a/scripts/build_sequestration_potentials.py
+++ b/scripts/build_sequestration_potentials.py
@@ -41,7 +41,7 @@ if __name__ == "__main__":
             "build_sequestration_potentials", simpl="", clusters="181"
         )
 
-    cf = snakemake.params["co2seq_potential"]
+    cf = snakemake.params.co2seq_potential
 
     gdf = gpd.read_file(snakemake.input.sequestration_potential[0])
 
diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py
index 0c8b0a94..eb837409 100644
--- a/scripts/build_shapes.py
+++ b/scripts/build_shapes.py
@@ -255,13 +255,11 @@ if __name__ == "__main__":
         snakemake = mock_snakemake("build_shapes")
     configure_logging(snakemake)
 
-    country_shapes = countries(
-        snakemake.input.naturalearth, snakemake.params["countries"]
-    )
+    country_shapes = countries(snakemake.input.naturalearth, snakemake.params.countries)
     country_shapes.reset_index().to_file(snakemake.output.country_shapes)
 
     offshore_shapes = eez(
-        country_shapes, snakemake.input.eez, snakemake.params["countries"]
+        country_shapes, snakemake.input.eez, snakemake.params.countries
     )
     offshore_shapes.reset_index().to_file(snakemake.output.offshore_shapes)
 
diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py
index 180007b7..d285691a 100644
--- a/scripts/build_solar_thermal_profiles.py
+++ b/scripts/build_solar_thermal_profiles.py
@@ -27,9 +27,9 @@ if __name__ == "__main__":
     cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1)
     client = Client(cluster, asynchronous=True)
 
-    config = snakemake.params["solar_thermal"]
+    config = snakemake.params.solar_thermal
 
-    time = pd.date_range(freq="h", **snakemake.params["snapshots"])
+    time = pd.date_range(freq="h", **snakemake.params.snapshots)
     cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time)
 
     clustered_regions = (
diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py
index ee06aebb..9db37c25 100644
--- a/scripts/build_temperature_profiles.py
+++ b/scripts/build_temperature_profiles.py
@@ -27,7 +27,7 @@ if __name__ == "__main__":
     cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1)
     client = Client(cluster, asynchronous=True)
 
-    time = pd.date_range(freq="h", **snakemake.params["snapshots"])
+    time = pd.date_range(freq="h", **snakemake.params.snapshots)
     cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time)
 
     clustered_regions = (
diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py
index dc2b94b9..c5bf4632 100644
--- a/scripts/build_transport_demand.py
+++ b/scripts/build_transport_demand.py
@@ -175,9 +175,9 @@ if __name__ == "__main__":
         snakemake.input.pop_weighted_energy_totals, index_col=0
     )
 
-    options = snakemake.params["sector"]
+    options = snakemake.params.sector
 
-    snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"], tz="UTC")
+    snapshots = pd.date_range(freq="h", **snakemake.params.snapshots, tz="UTC")
 
     nyears = len(snapshots) / 8760
 
diff --git a/scripts/make_summary.py b/scripts/make_summary.py
index da0712d7..00fca2fd 100644
--- a/scripts/make_summary.py
+++ b/scripts/make_summary.py
@@ -198,7 +198,7 @@ def calculate_costs(n, label, costs):
 
 
 def calculate_cumulative_cost():
-    planning_horizons = snakemake.params["scenario"]["planning_horizons"]
+    planning_horizons = snakemake.params.scenario["planning_horizons"]
 
     cumulative_cost = pd.DataFrame(
         index=df["costs"].sum().index,
@@ -688,19 +688,19 @@ if __name__ == "__main__":
         (cluster, ll, opt + sector_opt, planning_horizon): "results/"
         + snakemake.params.RDIR
         + f"/postnetworks/elec_s{simpl}_{cluster}_l{ll}_{opt}_{sector_opt}_{planning_horizon}.nc"
-        for simpl in snakemake.params["scenario"]["simpl"]
-        for cluster in snakemake.params["scenario"]["clusters"]
-        for opt in snakemake.params["scenario"]["opts"]
-        for sector_opt in snakemake.params["scenario"]["sector_opts"]
-        for ll in snakemake.params["scenario"]["ll"]
-        for planning_horizon in snakemake.params["scenario"]["planning_horizons"]
+        for simpl in snakemake.params.scenario["simpl"]
+        for cluster in snakemake.params.scenario["clusters"]
+        for opt in snakemake.params.scenario["opts"]
+        for sector_opt in snakemake.params.scenario["sector_opts"]
+        for ll in snakemake.params.scenario["ll"]
+        for planning_horizon in snakemake.params.scenario["planning_horizons"]
     }
 
-    Nyears = len(pd.date_range(freq="h", **snakemake.params["snapshots"])) / 8760
+    Nyears = len(pd.date_range(freq="h", **snakemake.params.snapshots)) / 8760
 
     costs_db = prepare_costs(
         snakemake.input.costs,
-        snakemake.params["costs"],
+        snakemake.params.costs,
         Nyears,
     )
 
@@ -710,7 +710,7 @@ if __name__ == "__main__":
 
     to_csv(df)
 
-    if snakemake.params["foresight"] == "myopic":
+    if snakemake.params.foresight == "myopic":
         cumulative_cost = calculate_cumulative_cost()
         cumulative_cost.to_csv(
             "results/" + snakemake.params.RDIR + "/csvs/cumulative_cost.csv"
diff --git a/scripts/plot_network.py b/scripts/plot_network.py
index 399f46e8..184d86f0 100644
--- a/scripts/plot_network.py
+++ b/scripts/plot_network.py
@@ -70,7 +70,7 @@ def plot_map(
     transmission=False,
     with_legend=True,
 ):
-    tech_colors = snakemake.params["plotting"]["tech_colors"]
+    tech_colors = snakemake.params.plotting["tech_colors"]
 
     n = network.copy()
     assign_location(n)
@@ -116,9 +116,7 @@ def plot_map(
     costs = costs.stack()  # .sort_index()
 
     # hack because impossible to drop buses...
-    eu_location = snakemake.params["plotting"].get(
-        "eu_node_location", dict(x=-5.5, y=46)
-    )
+    eu_location = snakemake.params.plotting.get("eu_node_location", dict(x=-5.5, y=46))
     n.buses.loc["EU gas", "x"] = eu_location["x"]
     n.buses.loc["EU gas", "y"] = eu_location["y"]
 
@@ -315,7 +313,7 @@ def plot_h2_map(network, regions):
     h2_new = n.links[n.links.carrier == "H2 pipeline"]
     h2_retro = n.links[n.links.carrier == "H2 pipeline retrofitted"]
 
-    if snakemake.params["foresight"] == "myopic":
+    if snakemake.params.foresight == "myopic":
         # sum capacitiy for pipelines from different investment periods
         h2_new = group_pipes(h2_new)
 
@@ -558,7 +556,7 @@ def plot_ch4_map(network):
     link_widths_used = max_usage / linewidth_factor
     link_widths_used[max_usage < line_lower_threshold] = 0.0
 
-    tech_colors = snakemake.params["plotting"]["tech_colors"]
+    tech_colors = snakemake.params.plotting["tech_colors"]
 
     pipe_colors = {
         "gas pipeline": "#f08080",
@@ -700,7 +698,7 @@ def plot_map_without(network):
 
     # hack because impossible to drop buses...
     if "EU gas" in n.buses.index:
-        eu_location = snakemake.params["plotting"].get(
+        eu_location = snakemake.params.plotting.get(
             "eu_node_location", dict(x=-5.5, y=46)
         )
         n.buses.loc["EU gas", "x"] = eu_location["x"]
@@ -876,7 +874,7 @@ def plot_series(network, carrier="AC", name="test"):
             stacked=True,
             linewidth=0.0,
             color=[
-                snakemake.params["plotting"]["tech_colors"][i.replace(suffix, "")]
+                snakemake.params.plotting["tech_colors"][i.replace(suffix, "")]
                 for i in new_columns
             ],
         )
@@ -937,7 +935,7 @@ if __name__ == "__main__":
 
     regions = gpd.read_file(snakemake.input.regions).set_index("name")
 
-    map_opts = snakemake.params["plotting"]["map"]
+    map_opts = snakemake.params.plotting["map"]
 
     if map_opts["boundaries"] is None:
         map_opts["boundaries"] = regions.total_bounds[[0, 2, 1, 3]] + [-1, 1, -1, 1]
diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py
index 36f75207..e7de5473 100644
--- a/scripts/plot_summary.py
+++ b/scripts/plot_summary.py
@@ -142,7 +142,7 @@ def plot_costs():
 
     df = df.groupby(df.index.map(rename_techs)).sum()
 
-    to_drop = df.index[df.max(axis=1) < snakemake.params["plotting"]["costs_threshold"]]
+    to_drop = df.index[df.max(axis=1) < snakemake.params.plotting["costs_threshold"]]
 
     logger.info(
         f"Dropping technology with costs below {snakemake.params['plotting']['costs_threshold']} EUR billion per year"
@@ -165,7 +165,7 @@ def plot_costs():
         kind="bar",
         ax=ax,
         stacked=True,
-        color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index],
+        color=[snakemake.params.plotting["tech_colors"][i] for i in new_index],
     )
 
     handles, labels = ax.get_legend_handles_labels()
@@ -173,7 +173,7 @@ def plot_costs():
     handles.reverse()
     labels.reverse()
 
-    ax.set_ylim([0, snakemake.params["plotting"]["costs_max"]])
+    ax.set_ylim([0, snakemake.params.plotting["costs_max"]])
 
     ax.set_ylabel("System Cost [EUR billion per year]")
 
@@ -201,7 +201,7 @@ def plot_energy():
     df = df.groupby(df.index.map(rename_techs)).sum()
 
     to_drop = df.index[
-        df.abs().max(axis=1) < snakemake.params["plotting"]["energy_threshold"]
+        df.abs().max(axis=1) < snakemake.params.plotting["energy_threshold"]
     ]
 
     logger.info(
@@ -227,7 +227,7 @@ def plot_energy():
         kind="bar",
         ax=ax,
         stacked=True,
-        color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index],
+        color=[snakemake.params.plotting["tech_colors"][i] for i in new_index],
     )
 
     handles, labels = ax.get_legend_handles_labels()
@@ -237,8 +237,8 @@ def plot_energy():
 
     ax.set_ylim(
         [
-            snakemake.params["plotting"]["energy_min"],
-            snakemake.params["plotting"]["energy_max"],
+            snakemake.params.plotting["energy_min"],
+            snakemake.params.plotting["energy_max"],
         ]
     )
 
@@ -287,7 +287,7 @@ def plot_balances():
         df = df.groupby(df.index.map(rename_techs)).sum()
 
         to_drop = df.index[
-            df.abs().max(axis=1) < snakemake.params["plotting"]["energy_threshold"] / 10
+            df.abs().max(axis=1) < snakemake.params.plotting["energy_threshold"] / 10
         ]
 
         if v[0] in co2_carriers:
@@ -317,7 +317,7 @@ def plot_balances():
             kind="bar",
             ax=ax,
             stacked=True,
-            color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index],
+            color=[snakemake.params.plotting["tech_colors"][i] for i in new_index],
         )
 
         handles, labels = ax.get_legend_handles_labels()
@@ -455,10 +455,10 @@ def plot_carbon_budget_distribution(input_eurostat):
     ax1 = plt.subplot(gs1[0, 0])
     ax1.set_ylabel("CO$_2$ emissions (Gt per year)", fontsize=22)
     ax1.set_ylim([0, 5])
-    ax1.set_xlim([1990, snakemake.params["planning_horizons"][-1] + 1])
+    ax1.set_xlim([1990, snakemake.params.planning_horizons[-1] + 1])
 
     path_cb = "results/" + snakemake.params.RDIR + "/csvs/"
-    countries = snakemake.params["countries"]
+    countries = snakemake.params.countries
     e_1990 = co2_emissions_year(countries, input_eurostat, opts, year=1990)
     CO2_CAP = pd.read_csv(path_cb + "carbon_budget_distribution.csv", index_col=0)
 
@@ -555,7 +555,7 @@ if __name__ == "__main__":
 
     plot_balances()
 
-    for sector_opts in snakemake.params["sector_opts"]:
+    for sector_opts in snakemake.params.sector_opts:
         opts = sector_opts.split("-")
         for o in opts:
             if "cb" in o:
diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index 51777ef5..e48ee162 100755
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -253,12 +253,12 @@ if __name__ == "__main__":
     Nyears = n.snapshot_weightings.objective.sum() / 8760.0
     costs = load_costs(
         snakemake.input.tech_costs,
-        snakemake.params["costs"],
-        snakemake.params["max_hours"],
+        snakemake.params.costs,
+        snakemake.params.max_hours,
         Nyears,
     )
 
-    set_line_s_max_pu(n, snakemake.params["lines"]["s_max_pu"])
+    set_line_s_max_pu(n, snakemake.params.lines["s_max_pu"])
 
     for o in opts:
         m = re.match(r"^\d+h$", o, re.IGNORECASE)
@@ -269,7 +269,7 @@ if __name__ == "__main__":
     for o in opts:
         m = re.match(r"^\d+seg$", o, re.IGNORECASE)
         if m is not None:
-            solver_name = snakemake.params["solver_name"]
+            solver_name = snakemake.params.solver_name
             n = apply_time_segmentation(n, m.group(0)[:-3], solver_name)
             break
 
@@ -277,11 +277,11 @@ if __name__ == "__main__":
         if "Co2L" in o:
             m = re.findall("[0-9]*\.?[0-9]+$", o)
             if len(m) > 0:
-                co2limit = float(m[0]) * snakemake.params["co2base"]
+                co2limit = float(m[0]) * snakemake.params.co2base
                 add_co2limit(n, co2limit, Nyears)
                 logger.info("Setting CO2 limit according to wildcard value.")
             else:
-                add_co2limit(n, snakemake.params["co2limit"], Nyears)
+                add_co2limit(n, snakemake.params.co2limit, Nyears)
                 logger.info("Setting CO2 limit according to config value.")
             break
 
@@ -293,7 +293,7 @@ if __name__ == "__main__":
                 add_gaslimit(n, limit, Nyears)
                 logger.info("Setting gas usage limit according to wildcard value.")
             else:
-                add_gaslimit(n, snakemake.params["gaslimit"], Nyears)
+                add_gaslimit(n, snakemake.params.gaslimit, Nyears)
                 logger.info("Setting gas usage limit according to config value.")
             break
 
@@ -322,7 +322,7 @@ if __name__ == "__main__":
                 add_emission_prices(n, dict(co2=float(m[0])))
             else:
                 logger.info("Setting emission prices according to config value.")
-                add_emission_prices(n, snakemake.params["costs"]["emission_prices"])
+                add_emission_prices(n, snakemake.params.costs["emission_prices"])
             break
 
     ll_type, factor = snakemake.wildcards.ll[0], snakemake.wildcards.ll[1:]
@@ -330,8 +330,8 @@ if __name__ == "__main__":
 
     set_line_nom_max(
         n,
-        s_nom_max_set=snakemake.params["lines"].get("s_nom_max,", np.inf),
-        p_nom_max_set=snakemake.params["links"].get("p_nom_max,", np.inf),
+        s_nom_max_set=snakemake.params.lines.get("s_nom_max,", np.inf),
+        p_nom_max_set=snakemake.params.links.get("p_nom_max,", np.inf),
     )
 
     if "ATK" in opts:
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 8b8b7d80..77fab4f8 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -200,12 +200,12 @@ def co2_emissions_year(
     """
     Calculate CO2 emissions in one specific year (e.g. 1990 or 2018).
     """
-    emissions_scope = snakemake.params["energy"]["emissions"]
+    emissions_scope = snakemake.params.energy["emissions"]
     eea_co2 = build_eea_co2(snakemake.input.co2, year, emissions_scope)
 
     # TODO: read Eurostat data from year > 2014
     # this only affects the estimation of CO2 emissions for BA, RS, AL, ME, MK
-    report_year = snakemake.params["energy"]["eurostat_report_year"]
+    report_year = snakemake.params.energy["eurostat_report_year"]
     if year > 2014:
         eurostat_co2 = build_eurostat_co2(
             input_eurostat, countries, report_year, year=2014
@@ -241,7 +241,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year):
         carbon_budget = float(o[o.find("cb") + 2 : o.find("ex")])
         r = float(o[o.find("ex") + 2 :])
 
-    countries = snakemake.params["countries"]
+    countries = snakemake.params.countries
 
     e_1990 = co2_emissions_year(
         countries, input_eurostat, opts, emissions_scope, report_year, year=1990
@@ -252,7 +252,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year):
         countries, input_eurostat, opts, emissions_scope, report_year, year=2018
     )
 
-    planning_horizons = snakemake.params["planning_horizons"]
+    planning_horizons = snakemake.params.planning_horizons
     t_0 = planning_horizons[0]
 
     if "be" in o:
@@ -391,7 +391,7 @@ def update_wind_solar_costs(n, costs):
         with xr.open_dataset(profile) as ds:
             underwater_fraction = ds["underwater_fraction"].to_pandas()
             connection_cost = (
-                snakemake.params["length_factor"]
+                snakemake.params.length_factor
                 * ds["average_distance"].to_pandas()
                 * (
                     underwater_fraction
@@ -483,8 +483,8 @@ def remove_elec_base_techs(n):
     batteries and H2) from base electricity-only network, since they're added
     here differently using links.
     """
-    for c in n.iterate_components(snakemake.params["pypsa_eur"]):
-        to_keep = snakemake.params["pypsa_eur"][c.name]
+    for c in n.iterate_components(snakemake.params.pypsa_eur):
+        to_keep = snakemake.params.pypsa_eur[c.name]
         to_remove = pd.Index(c.df.carrier.unique()).symmetric_difference(to_keep)
         if to_remove.empty:
             continue
@@ -674,7 +674,7 @@ def add_dac(n, costs):
 def add_co2limit(n, nyears=1.0, limit=0.0):
     logger.info(f"Adding CO2 budget limit as per unit of 1990 levels of {limit}")
 
-    countries = snakemake.params["countries"]
+    countries = snakemake.params.countries
 
     sectors = emission_sectors_from_opts(opts)
 
@@ -787,7 +787,7 @@ def add_ammonia(n, costs):
 
     nodes = pop_layout.index
 
-    cf_industry = snakemake.params["industry"]
+    cf_industry = snakemake.params.industry
 
     n.add("Carrier", "NH3")
 
@@ -1102,7 +1102,7 @@ def add_storage_and_grids(n, costs):
             lifetime=costs.at["OCGT", "lifetime"],
         )
 
-    cavern_types = snakemake.params["sector"]["hydrogen_underground_storage_locations"]
+    cavern_types = snakemake.params.sector["hydrogen_underground_storage_locations"]
     h2_caverns = pd.read_csv(snakemake.input.h2_cavern, index_col=0)
 
     if (
@@ -3274,7 +3274,7 @@ if __name__ == "__main__":
 
     update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts)
 
-    options = snakemake.params["sector"]
+    options = snakemake.params.sector
 
     opts = snakemake.wildcards.sector_opts.split("-")
 
@@ -3289,7 +3289,7 @@ if __name__ == "__main__":
 
     costs = prepare_costs(
         snakemake.input.costs,
-        snakemake.params["costs"],
+        snakemake.params.costs,
         nyears,
     )
 
@@ -3301,10 +3301,10 @@ if __name__ == "__main__":
 
     spatial = define_spatial(pop_layout.index, options)
 
-    if snakemake.params["foresight"] == "myopic":
+    if snakemake.params.foresight == "myopic":
         add_lifetime_wind_solar(n, costs)
 
-        conventional = snakemake.params["conventional_carriers"]
+        conventional = snakemake.params.conventional_carriers
         for carrier in conventional:
             add_carrier_buses(n, carrier)
 
@@ -3369,19 +3369,19 @@ if __name__ == "__main__":
     if options["allam_cycle"]:
         add_allam(n, costs)
 
-    solver_name = snakemake.params["solver_name"]
+    solver_name = snakemake.params.solver_name
     n = set_temporal_aggregation(n, opts, solver_name)
 
     limit_type = "config"
-    limit = get(snakemake.params["co2_budget"], investment_year)
+    limit = get(snakemake.params.co2_budget, investment_year)
     for o in opts:
         if "cb" not in o:
             continue
         limit_type = "carbon budget"
         fn = "results/" + snakemake.params.RDIR + "/csvs/carbon_budget_distribution.csv"
         if not os.path.exists(fn):
-            emissions_scope = snakemake.params["emissions_scope"]
-            report_year = snakemake.params["report_year"]
+            emissions_scope = snakemake.params.emissions_scope
+            report_year = snakemake.params.report_year
             build_carbon_budget(
                 o, snakemake.input.eurostat, fn, emissions_scope, report_year
             )
@@ -3416,8 +3416,8 @@ if __name__ == "__main__":
     if options["electricity_grid_connection"]:
         add_electricity_grid_connection(n, costs)
 
-    first_year_myopic = (snakemake.params["foresight"] == "myopic") and (
-        snakemake.params["planning_horizons"][0] == investment_year
+    first_year_myopic = (snakemake.params.foresight == "myopic") and (
+        snakemake.params.planning_horizons[0] == investment_year
     )
 
     if options.get("cluster_heat_buses", False) and not first_year_myopic:
diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py
index 2204ac36..c7053bed 100644
--- a/scripts/retrieve_databundle.py
+++ b/scripts/retrieve_databundle.py
@@ -53,7 +53,7 @@ if __name__ == "__main__":
         snakemake
     )  # TODO Make logging compatible with progressbar (see PR #102)
 
-    if snakemake.params["tutorial"]:
+    if snakemake.params.tutorial:
         url = "https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz"
     else:
         url = "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz"
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index 41679a03..d80e38a0 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -44,9 +44,9 @@ pypsa.pf.logger.setLevel(logging.WARNING)
 from pypsa.descriptors import get_switchable_as_dense as get_as_dense
 
 
-def add_land_use_constraint(n, planning_horizons_param, config):
+def add_land_use_constraint(n, planning_horizons, config):
     if "m" in snakemake.wildcards.clusters:
-        _add_land_use_constraint_m(n, planning_horizons_param, config)
+        _add_land_use_constraint_m(n, planning_horizons, config)
     else:
         _add_land_use_constraint(n)
 
@@ -80,7 +80,7 @@ def _add_land_use_constraint(n):
     n.generators.p_nom_max.clip(lower=0, inplace=True)
 
 
-def _add_land_use_constraint_m(n, planning_horizons_param, config):
+def _add_land_use_constraint_m(n, planning_horizons, config):
     # if generators clustering is lower than network clustering, land_use accounting is at generators clusters
 
     planning_horizons = param["planning_horizons"]
@@ -145,9 +145,9 @@ def prepare_network(
     n,
     solve_opts=None,
     config=None,
-    foresight_param=None,
-    planning_horizons_param=None,
-    co2_sequestration_potential_param=None,
+    foresight=None,
+    planning_horizons=None,
+    co2_sequestration_potential=None,
 ):
     if "clip_p_max_pu" in solve_opts:
         for df in (
@@ -198,11 +198,11 @@ def prepare_network(
         n.set_snapshots(n.snapshots[:nhours])
         n.snapshot_weightings[:] = 8760.0 / nhours
 
-    if foresight_param == "myopic":
-        add_land_use_constraint(n, planning_horizons_param, config)
+    if foresight == "myopic":
+        add_land_use_constraint(n, planning_horizons, config)
 
     if n.stores.carrier.eq("co2 stored").any():
-        limit = co2_sequestration_potential_param
+        limit = co2_sequestration_potential
         add_co2_sequestration_limit(n, limit=limit)
 
     return n
@@ -597,13 +597,11 @@ def extra_functionality(n, snapshots):
     add_pipe_retrofit_constraint(n)
 
 
-def solve_network(n, config, solving_param, opts="", **kwargs):
-    set_of_options = solving_param["solver"]["options"]
-    solver_options = (
-        solving_param["solver_options"][set_of_options] if set_of_options else {}
-    )
-    solver_name = solving_param["solver"]["name"]
-    cf_solving = solving_param["options"]
+def solve_network(n, config, solving, opts="", **kwargs):
+    set_of_options = solving["solver"]["options"]
+    solver_options = solving["solver_options"][set_of_options] if set_of_options else {}
+    solver_name = solving["solver"]["name"]
+    cf_solving = solving["options"]
     track_iterations = cf_solving.get("track_iterations", False)
     min_iterations = cf_solving.get("min_iterations", 4)
     max_iterations = cf_solving.get("max_iterations", 6)
@@ -672,7 +670,7 @@ if __name__ == "__main__":
     if "sector_opts" in snakemake.wildcards.keys():
         opts += "-" + snakemake.wildcards.sector_opts
     opts = [o for o in opts.split("-") if o != ""]
-    solve_opts = snakemake.params["solving"]["options"]
+    solve_opts = snakemake.params.solving["options"]
 
     np.random.seed(solve_opts.get("seed", 123))
 
@@ -686,17 +684,15 @@ if __name__ == "__main__":
         n,
         solve_opts,
         config=snakemake.config,
-        foresight_param=snakemake.params["foresight"],
-        planning_horizons_param=snakemake.params["planning_horizons"],
-        co2_sequestration_potential_param=snakemake.params[
-            "co2_sequestration_potential"
-        ],
+        foresight=snakemake.params.foresight,
+        planning_horizons=snakemake.params.planning_horizons,
+        co2_sequestration_potential=snakemake.params["co2_sequestration_potential"],
     )
 
     n = solve_network(
         n,
         config=snakemake.config,
-        solving_param=snakemake.params["solving"],
+        solving=snakemake.params.solving,
         opts=opts,
         log_fn=snakemake.log.solver,
     )
diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py
index 421bc515..37e853e5 100644
--- a/scripts/solve_operations_network.py
+++ b/scripts/solve_operations_network.py
@@ -41,7 +41,7 @@ if __name__ == "__main__":
 
     opts = (snakemake.wildcards.opts + "-" + snakemake.wildcards.sector_opts).split("-")
     opts = [o for o in opts if o != ""]
-    solve_opts = snakemake.params["options"]
+    solve_opts = snakemake.params.options
 
     np.random.seed(solve_opts.get("seed", 123))
 

From 27eea273bd20559dcf02d32ed0638c2938fb4acf Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Thu, 15 Jun 2023 19:12:30 +0200
Subject: [PATCH 128/176] harmonize params names

---
 rules/build_electricity.smk               |  9 +++------
 rules/build_sector.smk                    |  7 ++++---
 scripts/add_electricity.py                | 21 ++++++---------------
 scripts/build_sequestration_potentials.py |  2 +-
 scripts/prepare_network.py                |  2 +-
 scripts/prepare_sector_network.py         |  4 ++--
 6 files changed, 17 insertions(+), 28 deletions(-)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index 9d4315df..44ebf404 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -320,11 +320,10 @@ rule simplify_network:
         aggregation_strategies=config["clustering"].get("aggregation_strategies", {}),
         focus_weights=config.get("focus_weights", None),
         renewable_carriers=config["electricity"]["renewable_carriers"],
-        costs=config["costs"],
         max_hours=config["electricity"]["max_hours"],
         length_factor=config["lines"]["length_factor"],
         p_max_pu=config["links"].get("p_max_pu", 1.0),
-        solver_name=config["solving"]["solver"]["name"],
+        costs=config["costs"],
     input:
         network=RESOURCES + "networks/elec.nc",
         tech_costs=COSTS,
@@ -357,10 +356,9 @@ rule cluster_network:
         focus_weights=config.get("focus_weights", None),
         renewable_carriers=config["electricity"]["renewable_carriers"],
         conventional_carriers=config["electricity"].get("conventional_carriers", []),
-        costs=config["costs"],
         max_hours=config["electricity"]["max_hours"],
         length_factor=config["lines"]["length_factor"],
-        solver_name=config["solving"]["solver"]["name"],
+        costs=config["costs"],
     input:
         network=RESOURCES + "networks/elec_s{simpl}.nc",
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson",
@@ -393,9 +391,9 @@ rule cluster_network:
 
 rule add_extra_components:
     params:
-        costs=config["costs"],
         extendable_carriers=config["electricity"]["extendable_carriers"],
         max_hours=config["electricity"]["max_hours"],
+        costs=config["costs"],
     input:
         network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc",
         tech_costs=COSTS,
@@ -418,7 +416,6 @@ rule prepare_network:
     params:
         links=config["links"],
         lines=config["lines"],
-        solver_name=config["solving"]["solver"]["name"],
         co2base=config["electricity"]["co2base"],
         co2limit=config["electricity"]["co2limit"],
         gaslimit=config["electricity"].get("gaslimit"),
diff --git a/rules/build_sector.smk b/rules/build_sector.smk
index e15394ff..1c3507fd 100644
--- a/rules/build_sector.smk
+++ b/rules/build_sector.smk
@@ -330,7 +330,9 @@ if config["sector"]["regional_co2_sequestration_potential"]["enable"]:
 
     rule build_sequestration_potentials:
         params:
-            co2seq_potential=config["sector"]["regional_co2_sequestration_potential"],
+            sequestration_potential=config["sector"][
+                "regional_co2_sequestration_potential"
+            ],
         input:
             sequestration_potential=HTTP.remote(
                 "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson",
@@ -705,7 +707,6 @@ rule build_transport_demand:
 rule prepare_sector_network:
     params:
         co2_budget=config["co2_budget"],
-        solver_name=config["solving"]["solver"]["name"],
         conventional_carriers=config["existing_capacities"]["conventional_carriers"],
         foresight=config["foresight"],
         costs=config["costs"],
@@ -716,7 +717,7 @@ rule prepare_sector_network:
         planning_horizons=config["scenario"]["planning_horizons"],
         countries=config["countries"],
         emissions_scope=config["energy"]["emissions"],
-        report_year=config["energy"]["eurostat_report_year"],
+        eurostat_report_year=config["energy"]["eurostat_report_year"],
         RDIR=RDIR,
     input:
         **build_retro_cost_output,
diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index d98dc767..e2f7fb65 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -137,7 +137,7 @@ def _add_missing_carriers_from_costs(n, costs, carriers):
     n.import_components_from_dataframe(emissions, "Carrier")
 
 
-def load_costs(tech_costs, params, max_hours, Nyears=1.0):
+def load_costs(tech_costs, costs, max_hours, Nyears=1.0):
     # set all asset costs and other parameters
     costs = pd.read_csv(tech_costs, index_col=[0, 1]).sort_index()
 
@@ -145,7 +145,7 @@ def load_costs(tech_costs, params, max_hours, Nyears=1.0):
     costs.loc[costs.unit.str.contains("/kW"), "value"] *= 1e3
     costs.unit = costs.unit.str.replace("/kW", "/MW")
 
-    fill_values = params["fill_values"]
+    fill_values = costs["fill_values"]
     costs = costs.value.unstack().fillna(fill_values)
 
     costs["capital_cost"] = (
@@ -168,8 +168,8 @@ def load_costs(tech_costs, params, max_hours, Nyears=1.0):
     costs.at["CCGT", "co2_emissions"] = costs.at["gas", "co2_emissions"]
 
     costs.at["solar", "capital_cost"] = (
-        params["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"]
-        + (1 - params["rooftop_share"]) * costs.at["solar-utility", "capital_cost"]
+        costs["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"]
+        + (1 - costs["rooftop_share"]) * costs.at["solar-utility", "capital_cost"]
     )
 
     def costs_for_storage(store, link1, link2=None, max_hours=1.0):
@@ -193,7 +193,7 @@ def load_costs(tech_costs, params, max_hours, Nyears=1.0):
     )
 
     for attr in ("marginal_cost", "capital_cost"):
-        overwrites = params.get(attr)
+        overwrites = costs.get(attr)
         if overwrites is not None:
             overwrites = pd.Series(overwrites)
             costs.loc[overwrites.index, attr] = overwrites
@@ -728,16 +728,6 @@ if __name__ == "__main__":
     )
     ppl = load_powerplants(snakemake.input.powerplants)
 
-    if "renewable_carriers" in params.electricity:
-        renewable_carriers = set(params.electricity["renewable_carriers"])
-    else:
-        logger.warning(
-            "Missing key `renewable_carriers` under config entry `electricity`. "
-            "In future versions, this will raise an error. "
-            "Falling back to carriers listed under `renewable`."
-        )
-        renewable_carriers = params.renewable
-
     attach_load(
         n,
         snakemake.input.regions,
@@ -749,6 +739,7 @@ if __name__ == "__main__":
 
     update_transmission_costs(n, costs, params.length_factor)
 
+    renewable_carriers = set(params.electricity["renewable_carriers"])
     extendable_carriers = params.electricity["extendable_carriers"]
     conventional_carriers = params.electricity["conventional_carriers"]
     conventional_inputs = {
diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py
index 0e59e55b..e19a96da 100644
--- a/scripts/build_sequestration_potentials.py
+++ b/scripts/build_sequestration_potentials.py
@@ -41,7 +41,7 @@ if __name__ == "__main__":
             "build_sequestration_potentials", simpl="", clusters="181"
         )
 
-    cf = snakemake.params.co2seq_potential
+    cf = snakemake.params.sequestration_potential
 
     gdf = gpd.read_file(snakemake.input.sequestration_potential[0])
 
diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index e48ee162..adbfa36c 100755
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -269,7 +269,7 @@ if __name__ == "__main__":
     for o in opts:
         m = re.match(r"^\d+seg$", o, re.IGNORECASE)
         if m is not None:
-            solver_name = snakemake.params.solver_name
+            solver_name = snakemake.config["solving"]["solver"]["name"]
             n = apply_time_segmentation(n, m.group(0)[:-3], solver_name)
             break
 
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 77fab4f8..73adf522 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -3369,7 +3369,7 @@ if __name__ == "__main__":
     if options["allam_cycle"]:
         add_allam(n, costs)
 
-    solver_name = snakemake.params.solver_name
+    solver_name = snakemake.config["solving"]["solver"]["name"]
     n = set_temporal_aggregation(n, opts, solver_name)
 
     limit_type = "config"
@@ -3381,7 +3381,7 @@ if __name__ == "__main__":
         fn = "results/" + snakemake.params.RDIR + "/csvs/carbon_budget_distribution.csv"
         if not os.path.exists(fn):
             emissions_scope = snakemake.params.emissions_scope
-            report_year = snakemake.params.report_year
+            report_year = snakemake.params.eurostat_report_year
             build_carbon_budget(
                 o, snakemake.input.eurostat, fn, emissions_scope, report_year
             )

From a17647a07e3c5e93fcaa2a686e0f72c09ab0dbfc Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Thu, 15 Jun 2023 19:35:41 +0200
Subject: [PATCH 129/176] add_electricity: fix cost ref

---
 scripts/add_electricity.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index e2f7fb65..594988bd 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -137,7 +137,7 @@ def _add_missing_carriers_from_costs(n, costs, carriers):
     n.import_components_from_dataframe(emissions, "Carrier")
 
 
-def load_costs(tech_costs, costs, max_hours, Nyears=1.0):
+def load_costs(tech_costs, config, max_hours, Nyears=1.0):
     # set all asset costs and other parameters
     costs = pd.read_csv(tech_costs, index_col=[0, 1]).sort_index()
 
@@ -145,7 +145,7 @@ def load_costs(tech_costs, costs, max_hours, Nyears=1.0):
     costs.loc[costs.unit.str.contains("/kW"), "value"] *= 1e3
     costs.unit = costs.unit.str.replace("/kW", "/MW")
 
-    fill_values = costs["fill_values"]
+    fill_values = config["fill_values"]
     costs = costs.value.unstack().fillna(fill_values)
 
     costs["capital_cost"] = (
@@ -168,8 +168,8 @@ def load_costs(tech_costs, costs, max_hours, Nyears=1.0):
     costs.at["CCGT", "co2_emissions"] = costs.at["gas", "co2_emissions"]
 
     costs.at["solar", "capital_cost"] = (
-        costs["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"]
-        + (1 - costs["rooftop_share"]) * costs.at["solar-utility", "capital_cost"]
+        config["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"]
+        + (1 - config["rooftop_share"]) * costs.at["solar-utility", "capital_cost"]
     )
 
     def costs_for_storage(store, link1, link2=None, max_hours=1.0):
@@ -193,7 +193,7 @@ def load_costs(tech_costs, costs, max_hours, Nyears=1.0):
     )
 
     for attr in ("marginal_cost", "capital_cost"):
-        overwrites = costs.get(attr)
+        overwrites = config.get(attr)
         if overwrites is not None:
             overwrites = pd.Series(overwrites)
             costs.loc[overwrites.index, attr] = overwrites

From 0bf693c585b66fb5e9e0faf020ecf3c6e63570c6 Mon Sep 17 00:00:00 2001
From: virio-andreyana <arsyan.virio@gmail.com>
Date: Fri, 16 Jun 2023 01:31:59 +0200
Subject: [PATCH 130/176] general update of .rst files

---
 config/config.default.yaml      | 145 ++++++++++------------
 doc/configtables/co2_budget.csv |   2 +
 doc/configtables/countries.csv  |   2 +
 doc/configtables/foresight.csv  |   2 +
 doc/configtables/toplevel.csv   |   6 +-
 doc/configuration.rst           | 209 +++++++++++++++++++++++---------
 doc/costs.rst                   |   2 +-
 doc/foresight.rst               |  11 +-
 doc/preparation.rst             |   2 +-
 doc/tutorial.rst                |  14 +--
 doc/wildcards.rst               |  17 +++
 11 files changed, 254 insertions(+), 158 deletions(-)
 create mode 100644 doc/configtables/co2_budget.csv
 create mode 100644 doc/configtables/countries.csv
 create mode 100644 doc/configtables/foresight.csv

diff --git a/config/config.default.yaml b/config/config.default.yaml
index 78c1385c..dc04a925 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: CC0-1.0
 
+# docs in 
 version: 0.8.0
 tutorial: false
 
@@ -9,63 +10,51 @@ logging:
   level: INFO
   format: '%(levelname)s:%(name)s:%(message)s'
 
-run:
-  name: "" # use this to keep track of runs with different settings
-  disable_progressbar: false # set to true to disable the progressbar
-  shared_resources: false # set to true to share the default resources across runs
-  shared_cutouts: true # set to true to share the default cutout(s) across runs
+# docs in 
+run: 
+  name: ""
+  disable_progressbar: false
+  shared_resources: false
+  shared_cutouts: true
 
-foresight: overnight # options are overnight, myopic, perfect (perfect is not yet implemented)
-# if you use myopic or perfect foresight, set the investment years in "planning_horizons" below
+# docs in 
+foresight: overnight
 
-scenario:
+# docs in 
+# Wildcard docs in 
+scenario: 
   simpl:
   - ''
-  ll:  # allowed transmission line volume expansion, can be any float >= 1.0 with a prefix v|c (today) or "copt"
-  - v1.0
+  ll:  
   - v1.5
-  clusters:  # number of nodes in Europe, any integer between 37 (1 node per country-zone) and several hundred
+  clusters:
   - 37
   - 128
   - 256
   - 512
   - 1024
-  opts: # only relevant for PyPSA-Eur
+  opts: 
   - ''
-  sector_opts: # this is where the main scenario settings are
+  sector_opts:
   - Co2L0-3H-T-H-B-I-A-solar+p3-dist1
-  # to really understand the options here, look in scripts/prepare_sector_network.py
-  # Co2Lx specifies the CO2 target in x% of the 1990 values; default will give default (5%);
-  # Co2L0p25 will give 25% CO2 emissions; Co2Lm0p05 will give 5% negative emissions
-  # xH is the temporal resolution; 3H is 3-hourly, i.e. one snapshot every 3 hours
-  # single letters are sectors: T for land transport, H for building heating,
-  # B for biomass supply, I for industry, shipping and aviation,
-  # A for agriculture, forestry and fishing
-  # solar+c0.5 reduces the capital cost of solar to 50\% of reference value
-  # solar+p3 multiplies the available installable potential by factor 3
-  # seq400 sets the potential of CO2 sequestration to 400 Mt CO2 per year
-  # dist{n} includes distribution grids with investment cost of n times cost in data/costs.csv
-  # for myopic/perfect foresight cb states the carbon budget in GtCO2 (cumulative
-  # emissions throughout the transition path in the timeframe determined by the
-  # planning_horizons), be:beta decay; ex:exponential decay
-  # cb40ex0 distributes a carbon budget of 40 GtCO2 following an exponential
-  # decay with initial growth rate 0
-  planning_horizons: # investment years for myopic and perfect; for overnight, year of cost assumptions can be different and is defined under 'costs'
-  - 2050
-  # for example, set to
+  planning_horizons:
   # - 2020
   # - 2030
   # - 2040
-  # - 2050
-  # for myopic foresight
+  - 2050
 
-countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK']
+# docs in 
+countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 
+            'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 
+            'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK']
 
+# docs in 
 snapshots:
   start: "2013-01-01"
   end: "2014-01-01"
-  inclusive: 'left' # include start, not end
+  inclusive: 'left'
 
+# docs in 
 enable:
   prepare_links_p_nom: false
   retrieve_databundle: true
@@ -77,9 +66,7 @@ enable:
   retrieve_natura_raster: true
   custom_busmap: false
 
-# CO2 budget as a fraction of 1990 emissions
-# this is over-ridden if CO2Lx is set in sector_opts
-# this is also over-ridden if cb is set in sector_opts
+# docs in 
 co2_budget:
   2020: 0.701
   2025: 0.524
@@ -89,18 +76,19 @@ co2_budget:
   2045: 0.032
   2050: 0.000
 
+# docs in 
 electricity:
   voltages: [220., 300., 380.]
-  gaslimit: false # global gas usage limit of X MWh_th
-  co2limit: 7.75e+7 # 0.05 * 3.1e9*0.5
+  gaslimit: false
+  co2limit: 7.75e+7
   co2base: 1.487e+9
   agg_p_nom_limits: data/agg_p_nom_minmax.csv
 
-  operational_reserve: # like https://genxproject.github.io/GenX/dev/core/#Reserves
+  operational_reserve:
     activate: false
-    epsilon_load: 0.02 # share of total load
-    epsilon_vres: 0.02 # share of total renewable supply
-    contingency: 4000 # fixed capacity in MW
+    epsilon_load: 0.02
+    epsilon_vres: 0.02
+    contingency: 4000
 
   max_hours:
     battery: 6
@@ -112,7 +100,6 @@ electricity:
     Store: [battery, H2]
     Link: [] # H2 pipeline
 
-  # use pandas query strings here, e.g. Country not in ['Germany']
   powerplants_filter: (DateOut >= 2022 or DateOut != DateOut)
   # use pandas query strings here, e.g. Country in ['Germany']
   custom_powerplants: false
@@ -136,7 +123,7 @@ electricity:
       Onshore: [onwind]
       PV: [solar]
 
-
+# docs in
 atlite:
   default_cutout: europe-2013-era5
   nprocesses: 4
@@ -163,20 +150,16 @@ atlite:
       sarah_dir:
       features: [influx, temperature]
 
-
+# docs in
 renewable:
   onwind:
     cutout: europe-2013-era5
     resource:
       method: wind
       turbine: Vestas_V112_3MW
-    capacity_per_sqkm: 3 # ScholzPhd Tab 4.3.1: 10MW/km^2 and assuming 30% fraction of the already restricted
-    # area is available for installation of wind generators due to competing land use and likely public
-    # acceptance issues.
+    capacity_per_sqkm: 3 
     # correction_factor: 0.93
     corine:
-      # Scholz, Y. (2012). Renewable energy based electricity supply at low costs
-      #  development of the REMix model and application for Europe. ( p.42 / p.28)
       grid_codes: [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32]
       distance: 1000
       distance_grid_codes: [1, 2, 3, 4, 5, 6]
@@ -189,13 +172,8 @@ renewable:
     resource:
       method: wind
       turbine: NREL_ReferenceTurbine_5MW_offshore
-    capacity_per_sqkm: 2 # ScholzPhd Tab 4.3.1: 10MW/km^2 and assuming 20% fraction of the already restricted
-    # area is available for installation of wind generators due to competing land use and likely public
-    # acceptance issues.
+    capacity_per_sqkm: 2 
     correction_factor: 0.8855
-    # proxy for wake losses
-    # from 10.1016/j.energy.2018.08.153
-    # until done more rigorously in #153
     corine: [44, 255]
     natura: true
     ship_threshold: 400
@@ -209,13 +187,8 @@ renewable:
     resource:
       method: wind
       turbine: NREL_ReferenceTurbine_5MW_offshore
-    capacity_per_sqkm: 2 # ScholzPhd Tab 4.3.1: 10MW/km^2 and assuming 20% fraction of the already restricted
-    # area is available for installation of wind generators due to competing land use and likely public
-    # acceptance issues.
+    capacity_per_sqkm: 2
     correction_factor: 0.8855
-    # proxy for wake losses
-    # from 10.1016/j.energy.2018.08.153
-    # until done more rigorously in #153
     corine: [44, 255]
     natura: true
     ship_threshold: 400
@@ -232,14 +205,7 @@ renewable:
       orientation:
         slope: 35.
         azimuth: 180.
-    capacity_per_sqkm: 1.7 # ScholzPhd Tab 4.3.1: 170 MW/km^2 and assuming 1% of the area can be used for solar PV panels
-    # Correction factor determined by comparing uncorrected area-weighted full-load hours to those
-    # published in Supplementary Data to
-    # Pietzcker, Robert Carl, et al. "Using the sun to decarbonize the power
-    # sector -- The economic potential of photovoltaics and concentrating solar
-    # power." Applied Energy 135 (2014): 704-720.
-    # This correction factor of 0.854337 may be in order if using reanalysis data.
-    # for discussion refer to https://github.com/PyPSA/pypsa-eur/pull/304
+    capacity_per_sqkm: 1.7
     # correction_factor: 0.854337
     corine: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 26, 31, 32]
     natura: true
@@ -253,10 +219,12 @@ renewable:
     hydro_max_hours: "energy_capacity_totals_by_country" # one of energy_capacity_totals_by_country, estimate_by_large_installations or a float
     clip_min_inflow: 1.0
 
+# docs in
 conventional:
   nuclear:
     p_max_pu: "data/nuclear_p_max_pu.csv" # float of file name
 
+# docs in
 lines:
   types:
     220.: "Al/St 240/40 2-bundle 220.0"
@@ -267,24 +235,28 @@ lines:
   length_factor: 1.25
   under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity
 
+# docs in
 links:
   p_max_pu: 1.0
   p_nom_max: .inf
   include_tyndp: true
   under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity
 
+# docs in
 transformers:
   x: 0.1
   s_nom: 2000.
   type: ''
 
+# docs in
 load:
-  power_statistics: true # only for files from <2019; set false in order to get ENTSOE transparency data
-  interpolate_limit: 3 # data gaps up until this size are interpolated linearly
-  time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from
+  power_statistics: true
+  interpolate_limit: 3
+  time_shift_for_large_gaps: 1w
   manual_adjustments: true # false
   scaling_factor: 1.0
 
+# TODO: PyPSA-Eur merge issue in prepare_sector_network.py
 # regulate what components with which carriers are kept from PyPSA-Eur;
 # some technologies are removed because they are implemented differently
 # (e.g. battery or H2 storage) or have different year-dependent costs
@@ -305,12 +277,14 @@ pypsa_eur:
   - hydro
   Store: []
 
+# docs in
 energy:
   energy_totals_year: 2011
   base_emissions_year: 1990
   eurostat_report_year: 2016
   emissions: CO2 # "CO2" or "All greenhouse gases - (CO2 equivalent)"
 
+# docs in
 biomass:
   year: 2030
   scenario: ENS_Med
@@ -336,13 +310,14 @@ biomass:
     - Manure solid, liquid
     - Sludge
 
-
+# docs in
 solar_thermal:
   clearsky_model: simple  # should be "simple" or "enhanced"?
   orientation:
     slope: 45.
     azimuth: 180.
 
+# docs under construction in
 # only relevant for foresight = myopic or perfect
 existing_capacities:
   grouping_years_power: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030]
@@ -354,7 +329,7 @@ existing_capacities:
   - oil
   - uranium
 
-
+# docs under construction in
 sector:
   district_heating:
     potential: 0.6  # maximum fraction of urban demand which can be supplied by district heating
@@ -515,6 +490,7 @@ sector:
   biomass_to_liquid: false
   biosng: false
 
+# docs under construction in
 industry:
   St_primary_fraction:  # fraction of steel produced via primary route versus secondary route (scrap+EAF); today fraction is 0.6
     2020: 0.6
@@ -568,6 +544,7 @@ industry:
   # DECHEMA (2017): https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf
   # Material Economics (2019): https://materialeconomics.com/latest-updates/industrial-transformation-2050
 
+# docs in
 costs:
   year: 2030
   version: v0.5.0
@@ -594,14 +571,15 @@ costs:
     fuel cell: 0.
     battery: 0.
     battery inverter: 0.
-  emission_prices: # in currency per tonne emission, only used with the option Ep
+  emission_prices:
     co2: 0.
 
+# docs in
 clustering:
   simplify_network:
-    to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections)
+    to_substations: false
     algorithm: kmeans # choose from: [hac, kmeans]
-    feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc.
+    feature: solar+onwind-time
     exclude_carriers: []
     remove_stubs: true
     remove_stubs_across_borders: true
@@ -611,7 +589,7 @@ clustering:
     exclude_carriers: []
   aggregation_strategies:
     generators:
-      p_nom_max: sum # use "min" for more conservative assumptions
+      p_nom_max: sum
       p_nom_min: sum
       p_min_pu: mean
       marginal_cost: mean
@@ -620,6 +598,7 @@ clustering:
       ramp_limit_down: max
       efficiency: mean
 
+# docs in
 solving:
   #tmpdir: "path/to/tmp"
   options:
@@ -693,7 +672,7 @@ solving:
 
   mem: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2
 
-
+# docs in
 plotting:
   map:
     boundaries: [-11, 30, 34, 71]
diff --git a/doc/configtables/co2_budget.csv b/doc/configtables/co2_budget.csv
new file mode 100644
index 00000000..21b42f05
--- /dev/null
+++ b/doc/configtables/co2_budget.csv
@@ -0,0 +1,2 @@
+,Unit,Values,Description
+co2_budget,--,Dictionary with planning horizons as keys.,CO2 budget as a fraction of 1990 emissions. Overwritten if ``CO2Lx`` or ``cb`` are set in ``{sector_opts}`` wildcard"doc/configtables/othertoplevel.csv
diff --git a/doc/configtables/countries.csv b/doc/configtables/countries.csv
new file mode 100644
index 00000000..63dd85b3
--- /dev/null
+++ b/doc/configtables/countries.csv
@@ -0,0 +1,2 @@
+ ,Unit,Values,Description
+countries,--,"Subset of {'AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'}","European countries defined by their `Two-letter country codes (ISO 3166-1) <https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2>`_ which should be included in the energy system model."
\ No newline at end of file
diff --git a/doc/configtables/foresight.csv b/doc/configtables/foresight.csv
new file mode 100644
index 00000000..07f930f2
--- /dev/null
+++ b/doc/configtables/foresight.csv
@@ -0,0 +1,2 @@
+,Unit,Values,Description
+foresight,string,"{overnight, myopic, perfect}","See :ref:`Foresight Options` for detail explanations."
\ No newline at end of file
diff --git a/doc/configtables/toplevel.csv b/doc/configtables/toplevel.csv
index 8a4b443c..9496e165 100644
--- a/doc/configtables/toplevel.csv
+++ b/doc/configtables/toplevel.csv
@@ -3,8 +3,4 @@ version,--,0.x.x,"Version of PyPSA-Eur. Descriptive only."
 tutorial,bool,"{true, false}","Switch to retrieve the tutorial data set instead of the full data set."
 logging,,,
 -- level,--,"Any of {'INFO', 'WARNING', 'ERROR'}","Restrict console outputs to all infos, warning or errors only"
--- format,--,"","Custom format for log messages. See `LogRecord <https://docs.python.org/3/library/logging.html#logging.LogRecord>`_ attributes."
-foresight,string,"{overnight, myopic, perfect}","Defaults to overnight scenarios."
-countries,--,"Subset of {'AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'}","European countries defined by their `Two-letter country codes (ISO 3166-1) <https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2>`_ which should be included in the energy system model."
-focus_weights,--,"Keys should be two-digit country codes (e.g. DE) and values should range between 0 and 1","Ratio of total clusters for particular countries. the remaining weight is distributed according to mean load. An example: ``focus_weights: 'DE': 0.6 'FR': 0.2``."
-co2_budget,--,"Dictionary with planning horizons as keys.","CO2 budget as a fraction of 1990 emissions. Overwritten if ``CO2Lx`` or ``cb`` are set in ``{sector_opts}`` wildcard"
+-- format,--,"","Custom format for log messages. See `LogRecord <https://docs.python.org/3/library/logging.html#logging.LogRecord>`_ attributes."
\ No newline at end of file
diff --git a/doc/configuration.rst b/doc/configuration.rst
index ee61c018..89b6f42f 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -18,7 +18,8 @@ Top-level configuration
 
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
-   :lines: 5-11,18-19,62,80-90
+   :start-at: version:
+   :end-before: # docs
 
 
 .. csv-table::
@@ -26,7 +27,7 @@ Top-level configuration
    :widths: 25,7,22,30
    :file: configtables/toplevel.csv
 
-.. _scenario:
+.. _run_cf:
 
 ``run``
 =======
@@ -40,13 +41,34 @@ The ``run`` section is used for running and storing scenarios with different con
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: run:
-   :end-before: foresight:
+   :end-before: # docs
 
 .. csv-table::
    :header-rows: 1
    :widths: 25,7,22,30
    :file: configtables/run.csv
 
+.. _foresight_cf:
+
+``foresight``
+=============
+
+.. literalinclude:: ../config/config.default.yaml
+   :language: yaml
+   :start-at: foresight:
+   :end-at: foresight:
+
+.. csv-table::
+   :header-rows: 1
+   :widths: 25,7,22,30
+   :file: configtables/foresight.csv
+
+.. note::
+    if you use myopic or perfect foresight, The investment years in 
+    :ref:`planning_horizons` in scenario has to be set.
+
+.. _scenario:
+
 ``scenario``
 ============
 
@@ -79,13 +101,28 @@ An exemplary dependency graph (starting from the simplification rules) then look
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: scenario:
-   :end-before: countries:
+   :end-before: # docs
 
 .. csv-table::
    :header-rows: 1
    :widths: 25,7,22,30
    :file: configtables/scenario.csv
 
+.. _countries:
+
+``countries``
+=============
+
+.. literalinclude:: ../config/config.default.yaml
+   :language: yaml
+   :start-at: countries:
+   :end-before: # docs
+
+.. csv-table::
+   :header-rows: 1
+   :widths: 25,7,22,30
+   :file: configtables/countries.csv
+
 .. _snapshots_cf:
 
 ``snapshots``
@@ -96,7 +133,7 @@ Specifies the temporal range to build an energy system model for as arguments to
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: snapshots:
-   :end-before: enable:
+   :end-before: # docs
 
 .. csv-table::
    :header-rows: 1
@@ -113,13 +150,32 @@ Switches for some rules and optional features.
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: enable:
-   :end-before: co2_budget:
+   :end-before: # docs
 
 .. csv-table::
    :header-rows: 1
    :widths: 25,7,22,30
    :file: configtables/enable.csv
 
+.. _CO2_budget_cf:
+
+``co2 budget``
+==============
+
+.. literalinclude:: ../config/config.default.yaml
+   :language: yaml
+   :start-at: co2_budget:
+   :end-before: # docs
+
+.. csv-table::
+   :header-rows: 1
+   :widths: 25,7,22,30
+   :file: configtables/co2_budget.csv
+
+.. note::
+    this parameter is over-ridden if ``CO2Lx`` or ``cb`` is set in 
+    sector_opts.
+
 .. _electricity_cf:
 
 ``electricity``
@@ -128,7 +184,7 @@ Switches for some rules and optional features.
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: electricity:
-   :end-before: atlite:
+   :end-before: # docs
 
 .. csv-table::
    :header-rows: 1
@@ -145,7 +201,7 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: atlite:
-   :end-before: renewable:
+   :end-before: # docs
 
 .. csv-table::
    :header-rows: 1
@@ -170,6 +226,15 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia
    :widths: 25,7,22,30
    :file: configtables/onwind.csv
 
+.. note::
+   Notes on ``capacity_per_sqkm``. ScholzPhd Tab 4.3.1: 10MW/km^2 and assuming 30% fraction of the already restricted
+   area is available for installation of wind generators due to competing land use and likely public
+   acceptance issues.
+
+.. note::
+   The default choice for corine ``grid_codes`` was based on Scholz, Y. (2012). Renewable energy based electricity supply at low costs
+   development of the REMix model and application for Europe. ( p.42 / p.28)
+
 ``offwind-ac``
 --------------
 
@@ -183,6 +248,16 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia
    :widths: 25,7,22,30
    :file: configtables/offwind-ac.csv
 
+.. note::
+   Notes on ``capacity_per_sqkm``. ScholzPhd Tab 4.3.1: 10MW/km^2 and assuming 20% fraction of the already restricted
+   area is available for installation of wind generators due to competing land use and likely public
+   acceptance issues.
+
+.. note::
+   Notes on ``correction_factor``. Correction due to proxy for wake losses
+   from 10.1016/j.energy.2018.08.153
+   until done more rigorously in #153
+
 ``offwind-dc``
 ---------------
 
@@ -196,6 +271,10 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia
    :widths: 25,7,22,30
    :file: configtables/offwind-dc.csv
 
+.. note::
+   both ``offwind-ac`` and ``offwind-dc`` have the same assumption on 
+   ``capacity_per_sqkm`` and ``correction_factor``.
+
 ``solar``
 ---------------
 
@@ -209,13 +288,22 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia
    :widths: 25,7,22,30
    :file: configtables/solar.csv
 
+.. note::
+   Notes on ``capacity_per_sqkm``. ScholzPhd Tab 4.3.1: 170 MW/km^2 and assuming 1% of the area can be used for solar PV panels.
+   Correction factor determined by comparing uncorrected area-weighted full-load hours to those
+   published in Supplementary Data to Pietzcker, Robert Carl, et al. "Using the sun to decarbonize the power
+   sector -- The economic potential of photovoltaics and concentrating solar
+   power." Applied Energy 135 (2014): 704-720.
+   This correction factor of 0.854337 may be in order if using reanalysis data.
+   for discussion refer to this <issue https://github.com/PyPSA/pypsa-eur/issues/285>
+
 ``hydro``
 ---------------
 
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at:   hydro:
-   :end-before: conventional:
+   :end-before: # docs
 
 .. csv-table::
    :header-rows: 1
@@ -237,7 +325,7 @@ overwrite the existing values.
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at:   conventional:
-   :end-before: lines:
+   :end-before: # docs
 
 .. csv-table::
    :header-rows: 1
@@ -250,7 +338,7 @@ overwrite the existing values.
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: lines:
-   :end-before: links:
+   :end-before: # docs
 
 .. csv-table::
    :header-rows: 1
@@ -265,7 +353,7 @@ overwrite the existing values.
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: links:
-   :end-before: transformers:
+   :end-before: # docs
 
 .. csv-table::
    :header-rows: 1
@@ -280,7 +368,7 @@ overwrite the existing values.
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: transformers:
-   :end-before: load:
+   :end-before: # docs
 
 .. csv-table::
    :header-rows: 1
@@ -295,45 +383,13 @@ overwrite the existing values.
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-after:   type:
-   :end-at:   scaling_factor:
+   :end-before: # docs
 
 .. csv-table::
    :header-rows: 1
    :widths: 25,7,22,30
    :file: configtables/load.csv
 
-.. _costs_cf:
-
-``costs``
-=============
-
-.. literalinclude:: ../config/config.default.yaml
-   :language: yaml
-   :start-at: costs:
-   :end-before: clustering:
-
-.. csv-table::
-   :header-rows: 1
-   :widths: 25,7,22,30
-   :file: configtables/costs.csv
-
-
-.. _clustering_cf:
-
-``clustering``
-==============
-
-.. literalinclude:: ../config/config.default.yaml
-   :language: yaml
-   :start-at: clustering:
-   :end-before: solving:
-
-.. csv-table::
-   :header-rows: 1
-   :widths: 25,7,22,30
-   :file: configtables/clustering.csv
-
-
 .. _energy_cf:
 
 ``energy``
@@ -348,7 +404,7 @@ overwrite the existing values.
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: energy:
-   :end-before: biomass:
+   :end-before: # docs
 
 
 .. _biomass_cf:
@@ -365,7 +421,7 @@ overwrite the existing values.
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: biomass:
-   :end-before: solar_thermal:
+   :end-before: # docs
 
 .. _solar_thermal_cf:
 
@@ -381,7 +437,7 @@ overwrite the existing values.
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: solar_thermal:
-   :end-before: existing_capacities:
+   :end-before: # docs
 
 .. _existing_capacities_cf:
 
@@ -397,7 +453,7 @@ overwrite the existing values.
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: existing_capacities:
-   :end-before: sector:
+   :end-before: # docs
 
 .. _sector_cf:
 
@@ -413,7 +469,7 @@ overwrite the existing values.
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: sector:
-   :end-before: industry:
+   :end-before: # docs
 
 .. _industry_cf:
 
@@ -429,7 +485,49 @@ overwrite the existing values.
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: industry:
-   :end-before: costs:
+   :end-before: # docs
+
+.. _costs_cf:
+
+``costs``
+=============
+
+.. literalinclude:: ../config/config.default.yaml
+   :language: yaml
+   :start-at: costs:
+   :end-before: # docs
+
+.. csv-table::
+   :header-rows: 1
+   :widths: 25,7,22,30
+   :file: configtables/costs.csv
+
+.. note:: 
+   ``rooftop_share:`` are based on the potentials, assuming  
+   (0.1 kW/m2 and 10 m2/person)
+
+.. _clustering_cf:
+
+``clustering``
+==============
+
+.. literalinclude:: ../config/config.default.yaml
+   :language: yaml
+   :start-at: clustering:
+   :end-before: # docs
+
+.. csv-table::
+   :header-rows: 1
+   :widths: 25,7,22,30
+   :file: configtables/clustering.csv
+
+.. note:: 
+   ``feature:`` in ``simplify_network:`` 
+   are only relevant if ``hac`` were chosen in ``algorithm``.
+
+.. tip:: 
+   use ``min`` in ``p_nom_max:`` for more `
+   conservative assumptions.
 
 .. _solving_cf:
 
@@ -439,12 +537,7 @@ overwrite the existing values.
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: solving:
-   :end-before: plotting:
-
-.. csv-table::
-   :header-rows: 1
-   :widths: 25,7,22,30
-   :file: configtables/solving.csv
+   :end-before: # docs
 
 .. csv-table::
    :header-rows: 1
diff --git a/doc/costs.rst b/doc/costs.rst
index 21938ced..5ddbb360 100644
--- a/doc/costs.rst
+++ b/doc/costs.rst
@@ -12,7 +12,7 @@ The database of cost assumptions is retrieved from the repository
 saved to a file ``data/costs_{year}.csv``. The ``config/config.yaml`` provides options
 to choose a reference year and use a specific version of the repository.
 
-.. literalinclude:: ../config.default.yaml
+.. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: costs:
    :end-at:   version:
diff --git a/doc/foresight.rst b/doc/foresight.rst
index 9b821060..c1be3443 100644
--- a/doc/foresight.rst
+++ b/doc/foresight.rst
@@ -87,8 +87,12 @@ evolve with the myopic approach:
   vehicle-to-grid services.
 
 - The annual biomass potential (default year and scenario for which potential is
-  taken is 2030, defined `here
-  <https://github.com/PyPSA/pypsa-eur-sec/blob/413254e241fb37f55b41caba7264644805ad8e97/config.default.yaml#L109>`_)
+  taken is 2030, as defined in config)
+
+.. literalinclude:: ../config/test/config.myopic.yaml
+   :language: yaml
+   :start-at: biomass:
+   :end-at: year:
 
 
 Configuration
@@ -108,7 +112,7 @@ optimized. For a myopic optimization, this is equivalent to the investment year.
 To set the investment years which are sequentially simulated for the myopic
 investment planning, select for example:
 
-.. literalinclude:: ../test/config.myopic.yaml
+.. literalinclude:: ../config/test/config.myopic.yaml
    :language: yaml
    :start-at:   planning_horizons:
    :end-before: countries:
@@ -203,6 +207,7 @@ The myopic code solves the network for the time steps included in
    network comprises additional generator, storage, and link capacities with
    p_nom_extendable=True. The non-solved network is saved in
    ``results/run_name/networks/prenetworks-brownfield``.
+
 The base year is the first element in ``planning_horizons``. Step 1 is
 implemented with the rule add_baseyear for the base year and with the rule
 add_brownfield for the remaining planning_horizons.
diff --git a/doc/preparation.rst b/doc/preparation.rst
index b5a12062..5cdc8031 100644
--- a/doc/preparation.rst
+++ b/doc/preparation.rst
@@ -89,7 +89,7 @@ Rule ``build_powerplants``
 .. _electricity_demand:
 
 Rule ``build_electricity_demand``
-=============================
+==================================
 
 
 .. automodule:: build_electricity_demand
diff --git a/doc/tutorial.rst b/doc/tutorial.rst
index 1b87cefe..518c58a3 100644
--- a/doc/tutorial.rst
+++ b/doc/tutorial.rst
@@ -43,21 +43,21 @@ How to configure runs?
 
 The model can be adapted to only include selected countries (e.g. Belgium) instead of all European countries to limit the spatial scope.
 
-.. literalinclude:: ../test/config.electricity.yaml
+.. literalinclude:: ../config/test/config.electricity.yaml
    :language: yaml
    :start-at: countries:
    :end-before: snapshots:
 
 Likewise, the example's temporal scope can be restricted (e.g. to a single week).
 
-.. literalinclude:: ../test/config.electricity.yaml
+.. literalinclude:: ../config/test/config.electricity.yaml
    :language: yaml
    :start-at: snapshots:
    :end-before: electricity:
 
 It is also possible to allow less or more carbon-dioxide emissions. Here, we limit the emissions of Belgium to 100 Mt per year.
 
-.. literalinclude:: ../test/config.electricity.yaml
+.. literalinclude:: ../config/test/config.electricity.yaml
    :language: yaml
    :start-at: electricity:
    :end-before: extendable_carriers:
@@ -65,7 +65,7 @@ It is also possible to allow less or more carbon-dioxide emissions. Here, we lim
 PyPSA-Eur also includes a database of existing conventional powerplants.
 We can select which types of existing powerplants we like to be extendable:
 
-.. literalinclude:: ../test/config.electricity.yaml
+.. literalinclude:: ../config/test/config.electricity.yaml
    :language: yaml
    :start-at: extendable_carriers:
    :end-before: renewable_carriers:
@@ -74,7 +74,7 @@ To accurately model the temporal and spatial availability of renewables such as
 wind and solar energy, we rely on historical weather data. It is advisable to
 adapt the required range of coordinates to the selection of countries.
 
-.. literalinclude:: ../test/config.electricity.yaml
+.. literalinclude:: ../config/test/config.electricity.yaml
    :language: yaml
    :start-at: atlite:
    :end-before: renewable:
@@ -83,7 +83,7 @@ We can also decide which weather data source should be used to calculate
 potentials and capacity factor time-series for each carrier. For example, we may
 want to use the ERA-5 dataset for solar and not the default SARAH-2 dataset.
 
-.. literalinclude:: ../test/config.electricity.yaml
+.. literalinclude:: ../config/test/config.electricity.yaml
    :language: yaml
    :start-at: solar:
    :end-at: cutout:
@@ -91,7 +91,7 @@ want to use the ERA-5 dataset for solar and not the default SARAH-2 dataset.
 Finally, it is possible to pick a solver. For instance, this tutorial uses the
 open-source solver GLPK.
 
-.. literalinclude:: ../test/config.electricity.yaml
+.. literalinclude:: ../config/test/config.electricity.yaml
    :language: yaml
    :start-at: solver:
    :end-before: plotting:
diff --git a/doc/wildcards.rst b/doc/wildcards.rst
index 30c58929..75eec192 100644
--- a/doc/wildcards.rst
+++ b/doc/wildcards.rst
@@ -117,6 +117,23 @@ The ``{sector_opts}`` wildcard
 
 .. warning::
     More comprehensive documentation for this wildcard will be added soon.
+    To really understand the options here, look in scripts/prepare_sector_network.py
+
+  # Co2Lx specifies the CO2 target in x% of the 1990 values; default will give default (5%);
+  # Co2L0p25 will give 25% CO2 emissions; Co2Lm0p05 will give 5% negative emissions
+  # xH is the temporal resolution; 3H is 3-hourly, i.e. one snapshot every 3 hours
+  # single letters are sectors: T for land transport, H for building heating,
+  # B for biomass supply, I for industry, shipping and aviation,
+  # A for agriculture, forestry and fishing
+  # solar+c0.5 reduces the capital cost of solar to 50\% of reference value
+  # solar+p3 multiplies the available installable potential by factor 3
+  # seq400 sets the potential of CO2 sequestration to 400 Mt CO2 per year
+  # dist{n} includes distribution grids with investment cost of n times cost in data/costs.csv
+  # for myopic/perfect foresight cb states the carbon budget in GtCO2 (cumulative
+  # emissions throughout the transition path in the timeframe determined by the
+  # planning_horizons), be:beta decay; ex:exponential decay
+  # cb40ex0 distributes a carbon budget of 40 GtCO2 following an exponential
+  # decay with initial growth rate 0
 
 The ``{sector_opts}`` wildcard is only used for sector-coupling studies.
 

From 284e3f8e70c4c8f44346bb519967d82256d5b3b3 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 19 Jun 2023 22:43:01 +0000
Subject: [PATCH 131/176] [pre-commit.ci] pre-commit autoupdate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

updates:
- [github.com/codespell-project/codespell: v2.2.4 → v2.2.5](https://github.com/codespell-project/codespell/compare/v2.2.4...v2.2.5)
---
 .pre-commit-config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 826a4819..5acfbf42 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -30,7 +30,7 @@ repos:
 
   # Find common spelling mistakes in comments and docstrings
 - repo: https://github.com/codespell-project/codespell
-  rev: v2.2.4
+  rev: v2.2.5
   hooks:
   - id: codespell
     args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom,appartment,bage,ore,setis,tabacco,berfore'] # Ignore capital case words, e.g. country codes

From 678792009a7f85c4032add17bf8242e4d3f2f914 Mon Sep 17 00:00:00 2001
From: virio-andreyana <arsyan.virio@gmail.com>
Date: Thu, 22 Jun 2023 10:23:54 +0200
Subject: [PATCH 132/176] preparing sector config data

---
 doc/configtables/sector-opts.csv |  2 +-
 doc/configtables/sector.csv      | 26 ++++++++++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)
 create mode 100644 doc/configtables/sector.csv

diff --git a/doc/configtables/sector-opts.csv b/doc/configtables/sector-opts.csv
index 5a6b6852..ea39c3b0 100644
--- a/doc/configtables/sector-opts.csv
+++ b/doc/configtables/sector-opts.csv
@@ -1,5 +1,5 @@
 Trigger, Description, Definition, Status
-``nH``, i.e. ``2H``-``6H``, Resample the time-resolution by averaging over every ``n`` snapshots, ``prepare_network``: `average_every_nhours() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L110>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L146>`__), In active use
+``nH``, i.e. ``2H``-``6H``, "Resample the time-resolution by averaging over every ``n`` snapshots, ``prepare_network``: `average_every_nhours() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L110>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L146>`__)", In active use
 ``Co2L``, Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``. If a float is appended an overall emission limit relative to the emission level given in ``electricity: co2base`` is added (e.g. ``Co2L0.05`` limits emissisions to 5% of what is given in ``electricity: co2base``), ``prepare_network``: `add_co2limit() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L19>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L154>`__, In active use
 ``carrier+{c|p|m}factor``,"Alter the capital cost (``c``), installable potential (``p``) or marginal costs (``m``) of a carrier by a factor. Example: ``solar+c0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use
 ``T``,Add land transport sector,,In active use
diff --git a/doc/configtables/sector.csv b/doc/configtables/sector.csv
new file mode 100644
index 00000000..d553b7d9
--- /dev/null
+++ b/doc/configtables/sector.csv
@@ -0,0 +1,26 @@
+,Unit,Values,Description
+district_heating,,,`prepare_sector_network.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/prepare_sector_network.py>`_
+-- potential,,float,maximum fraction of urban demand which can be supplied by district heating increase of today's district heating demand to potential maximum district heating share
+-- progress,,Dictionary with planning horizons as keys.,Progress = 0 means today's district heating share. Progress = 1 means maximum fraction of urban demand is supplied by district heating
+-- district_heating_loss,,float,
+cluster_heat_buses,,bool,cluster residential and service heat buses in `prepare_sector_network.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/prepare_sector_network.py>`_  to one to save memory.
+bev_dsm_restriction_value,,float,Adding a stage of charge (SOC) limit  for battery electric vehicles (BEV) to manage its own energy demand (DSM). Located in `build_transport_demand.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/build_transport_demand.py>`_. 
+bev_dsm_restriction_time,,float,Time at which SOC of BEV has to be dsm_restriction_value
+transport_heating_deadband_upper,,,
+transport_heating_deadband_lower,,,
+ICE_lower_degree_factor,,,
+ICE_upper_degree_factor,,,
+EV_lower_degree_factor,,,
+EV_upper_degree_factor,,,
+bev_dsm,,,
+bev_availability,,,
+bev_energy,,,
+bev_charge_efficiency,,,
+bev_plug_to_wheel_efficiency,,,
+bev_charge_rate,,,
+bev_avail_max,,,
+bev_avail_mean,,,
+v2g,,,
+land_transport_fuel_cell_share,,,
+land_transport_electric_share,,,
+land_transport_ice_share,,,

From cec407e656bfa7c684b821d0e714ba03111158f4 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 26 Jun 2023 22:19:08 +0000
Subject: [PATCH 133/176] [pre-commit.ci] pre-commit autoupdate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

updates:
- [github.com/PyCQA/docformatter: v1.7.2 → v1.7.3](https://github.com/PyCQA/docformatter/compare/v1.7.2...v1.7.3)
- [github.com/fsfe/reuse-tool: v1.1.2 → v2.0.0](https://github.com/fsfe/reuse-tool/compare/v1.1.2...v2.0.0)
---
 .pre-commit-config.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 5acfbf42..7bb35b53 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -39,7 +39,7 @@ repos:
 
   # Make docstrings PEP 257 compliant
 - repo: https://github.com/PyCQA/docformatter
-  rev: v1.7.2
+  rev: v1.7.3
   hooks:
   - id: docformatter
     args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"]
@@ -87,6 +87,6 @@ repos:
 
   # Check for FSFE REUSE compliance (licensing)
 - repo: https://github.com/fsfe/reuse-tool
-  rev: v1.1.2
+  rev: v2.0.0
   hooks:
   - id: reuse

From d943e3bb6847b27e8ec8231b56337627e7064913 Mon Sep 17 00:00:00 2001
From: virio-andreyana <arsyan.virio@gmail.com>
Date: Tue, 27 Jun 2023 23:16:15 +0200
Subject: [PATCH 134/176] add sector and industry table. In progress

---
 doc/configtables/industry.csv |  28 +++++++
 doc/configtables/sector.csv   | 133 +++++++++++++++++++++++++++-------
 doc/configuration.rst         |  10 +++
 3 files changed, 146 insertions(+), 25 deletions(-)
 create mode 100644 doc/configtables/industry.csv

diff --git a/doc/configtables/industry.csv b/doc/configtables/industry.csv
new file mode 100644
index 00000000..6d503498
--- /dev/null
+++ b/doc/configtables/industry.csv
@@ -0,0 +1,28 @@
+,Unit,Values,Description
+St_primary_fraction,--,Dictionary with planning horizons as keys.,The fraction of steel produced via primary route versus secondary route (scrap+EAF). Current fraction is 0.6
+DRI_fraction,--,Dictionary with planning horizons as keys.,The fraction of the primary route converted to DRI + EAF
+H2_DRI,--,float,The hydrogen consumption in Direct Reduced Iron (DRI)  Mwh_H2 LHV/ton_Steel from 51kgH2/tSt in Vogl et al (2018) doi:10.1016/j.jclepro.2018.08.279
+elec_DRI,--,float,The electricity consumed in Direct Reduced Iron (DRI) shaft. MWh/tSt HYBRIT brochure https://ssabwebsitecdn.azureedge.net/-/media/hybrit/files/hybrit_brochure.pdf
+Al_primary_fraction,--,Dictionary with planning horizons as keys.,The fraction of aluminium produced via the primary route versus scrap.  Current fraction is 0.4
+MWh_NH3_per_tNH3,LHV,float,The energy amount per ton of ammonia.
+MWh_CH4_per_tNH3_SMR,--,float,The energy amount of methane needed to produce a ton of ammonia using steam methane reforming (SMR). Value derived from 2012's demand from https://ec.europa.eu/docsroom/documents/4165/attachments/1/translations/en/renditions/pdf
+MWh_elec_per_tNH3_SMR,--,float,"The energy amount of electricity needed to produce a ton of ammonia using steam methane reforming (SMR). same source, assuming 94-6% split methane-elec of total energy demand 11.5 MWh/tNH3"
+MWh_H2_per_tNH3_electrolysis,--,float,"The energy amount of hydrogen needed to produce a ton of ammonia using Haber–Bosch process. From https://doi.org/10.1016/j.joule.2018.04.017, around 0.197 tH2/tHN3 (>3/17 since some H2 lost and used for energy)"
+MWh_elec_per_tNH3_electrolysis,--,float,The energy amount of electricity needed to produce a ton of ammonia using Haber–Bosch process. From https://doi.org/10.1016/j.joule.2018.04.017 Table 13 (air separation and HB)
+MWh_NH3_per_MWh_H2_cracker,--,float,The energy amount of amonia needed to produce an energy amount hydrogen using ammonia cracker. https://github.com/euronion/trace/blob/44a5ff8401762edbef80eff9cfe5a47c8d3c8be4/data/efficiencies.csv
+NH3_process_emissions,MtCO2/a,float,The emission of ammonia production from steam methane reforming (SMR)
+petrochemical_process_emissions,MtCO2/a,float,The emission of petrochemical production
+HVC_primary_fraction,--,float,The fraction of today's high value chemicals (HVC) produced via primary route
+HVC_mechanical_recycling_fraction,--,float,The fraction of today's high value chemicals (HVC) produced using mechanical recycling
+HVC_chemical_recycling_fraction,--,float,The fraction of today's high value chemicals (HVC) produced using chemical recycling
+HVC_production_today,MtHVC/a,float,The amount of high value chemicals (HVC) produced
+MWh_elec_per_tHVC_mechanical_recycling,--,float,The energy amount of electricity needed to produce a ton of high value chemical (HVC) using mechanical recycling
+MWh_elec_per_tHVC_chemical_recycling,--,float,The energy amount of electricity needed to produce a ton of high value chemical (HVC) using chemical recycling
+chlorine_production_today,MtCl/a,float,The amount of chlorine produced
+MWh_elec_per_tCl,--,float,The energy amount of electricity needed to produce a ton of chlorine
+MWh_H2_per_tCl,--,float,The energy amount of hydrogen needed to produce a ton of chlorine
+methanol_production_today,MtMeOH/a,float,The amount of methanol produced
+MWh_elec_per_tMeOH,--,float,The energy amount of electricity needed to produce a ton of methanol
+MWh_CH4_per_tMeOH,--,float,The energy amount of methane needed to produce a ton of methanol
+hotmaps_locate_missing,--,true or false,Locate industrial sites without valid locations based on city and countries.
+reference_year,--,year,
diff --git a/doc/configtables/sector.csv b/doc/configtables/sector.csv
index d553b7d9..27a69809 100644
--- a/doc/configtables/sector.csv
+++ b/doc/configtables/sector.csv
@@ -1,26 +1,109 @@
 ,Unit,Values,Description
-district_heating,,,`prepare_sector_network.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/prepare_sector_network.py>`_
--- potential,,float,maximum fraction of urban demand which can be supplied by district heating increase of today's district heating demand to potential maximum district heating share
--- progress,,Dictionary with planning horizons as keys.,Progress = 0 means today's district heating share. Progress = 1 means maximum fraction of urban demand is supplied by district heating
--- district_heating_loss,,float,
-cluster_heat_buses,,bool,cluster residential and service heat buses in `prepare_sector_network.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/prepare_sector_network.py>`_  to one to save memory.
-bev_dsm_restriction_value,,float,Adding a stage of charge (SOC) limit  for battery electric vehicles (BEV) to manage its own energy demand (DSM). Located in `build_transport_demand.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/build_transport_demand.py>`_. 
-bev_dsm_restriction_time,,float,Time at which SOC of BEV has to be dsm_restriction_value
-transport_heating_deadband_upper,,,
-transport_heating_deadband_lower,,,
-ICE_lower_degree_factor,,,
-ICE_upper_degree_factor,,,
-EV_lower_degree_factor,,,
-EV_upper_degree_factor,,,
-bev_dsm,,,
-bev_availability,,,
-bev_energy,,,
-bev_charge_efficiency,,,
-bev_plug_to_wheel_efficiency,,,
-bev_charge_rate,,,
-bev_avail_max,,,
-bev_avail_mean,,,
-v2g,,,
-land_transport_fuel_cell_share,,,
-land_transport_electric_share,,,
-land_transport_ice_share,,,
+district_heating,--,,`prepare_sector_network.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/prepare_sector_network.py>`_
+-- potential,--,float,maximum fraction of urban demand which can be supplied by district heating increase of today's district heating demand to potential maximum district heating share
+-- progress,--,Dictionary with planning horizons as keys.,Progress = 0 means today's district heating share. Progress = 1 means maximum fraction of urban demand is supplied by district heating
+-- district_heating_loss,--,float,
+cluster_heat_buses,--,true or false,cluster residential and service heat buses in `prepare_sector_network.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/prepare_sector_network.py>`_  to one to save memory.
+bev_dsm_restriction_value,--,float,Adding a stage of charge (SOC) limit  for battery electric vehicles (BEV) to manage its own energy demand (DSM). Located in `build_transport_demand.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/build_transport_demand.py>`_. 
+bev_dsm_restriction_time,--,float,Time at which SOC of BEV has to be dsm_restriction_value
+transport_heating_deadband_upper,°C,float,"The minimum temperature in the vehicle. At lower temperatures, the energy required for heating in the vehicle increases."
+transport_heating_deadband_lower,°C,float,"The maximum temperature in the vehicle. At higher temperatures, the energy required for cooling in the vehicle increases."
+ICE_lower_degree_factor,--,float,Percentage increase in energy demand in internal combustion engine (ICE) for each degree difference between the cold environment and the minimum temperature.
+ICE_upper_degree_factor,--,float,Percentage increase in energy demand in internal combustion engine (ICE) for each degree difference between the hot environment and the maximum temperature.
+EV_lower_degree_factor,--,float,Percentage increase in energy demand in electric vehicles (EV) for each degree difference between the cold environment and the minimum temperature.
+EV_upper_degree_factor,--,float,Percentage increase in energy demand in electric vehicles (EV) for each degree difference between the hot environment and the maximum temperature.
+bev_dsm,--,true or false,Add the option for battery electric vehicles (BEV) to manage its own energy demand (DSM)
+bev_availability,--,float,The percentage for battery electric vehicles (BEV) that are able to do demand side management (DSM)
+bev_energy,--,float,The average size of battery electric vehicles (BEV) in MWh
+bev_charge_efficiency,--,float,Battery electric vehicles (BEV) charge and discharge efficiency
+bev_plug_to_wheel_efficiency,km/kWh,float,The distance battery electric vehicles (BEV) can travel in km per kWh of energy charge in battery.  Base value comes from Tesla Model S https://www.fueleconomy.gov/feg/
+bev_charge_rate,MWh,float,The power consumption for one electric vehicle (EV) in MWh. Value derived from 3-phase charger with 11 kW.
+bev_avail_max,--,float,The maximum percentage plugged-in availability for passenger electric vehicles.
+bev_avail_mean,--,float,The average percentage plugged-in availability for passenger electric vehicles.
+v2g,--,true or false,Allows feed-in to grid from EV battery
+land_transport_fuel_cell_share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses fuel cells in a given year
+land_transport_electric_share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses electric vehicles (EV) in a given year
+land_transport_ice_share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses internal combustion engines (ICE) in a given year
+transport_fuel_cell_efficiency,--,float,The H2 conversion efficiencies of fuel cells in transport
+transport_internal_combustion_efficiency,--,float,The oil conversion efficiencies of internal combustion engine (ICE) in transport
+agriculture_machinery_electric_share,--,float,The percentage for agricultural machinery that uses electricity
+agriculture_machinery_oil_share,--,float,The percentage for agricultural machinery that uses oil
+agriculture_machinery_fuel_efficiency,--,float,The efficiency of electric-powered machinery in the conversion of electricity to meet agricultural needs.
+agriculture_machinery_electric_efficiency,--,float,The efficiency of oil-powered machinery in the conversion of oil to meet agricultural needs.
+MWh_MeOH_per_MWh_H2,LHV,float,The energy amount of the produced methanol per energy amount of hydrogen. source: DECHEMA (2017): Low carbon energy and feedstock for the European chemical industry page 64.
+MWh_MeOH_per_tCO2,LHV,float,The energy amount of the produced methanol per ton of CO2
+MWh_MeOH_per_MWh_e,LHV,float,The energy amount of the produced methanol per energy amount of electricity
+shipping_hydrogen_liquefaction,--,true or false,Consider whether to include liquefaction costs for shipping H2 demand.
+shipping_hydrogen_share,--,Dictionary with planning horizons as keys.,The share of ships powered by hydrogen in a given year
+shipping_methanol_share,--,Dictionary with planning horizons as keys.,The share of ships powered by methanol in a given year
+shipping_oil_share,--,Dictionary with planning horizons as keys.,The share of ships powered by oil in a given year
+shipping_methanol_efficiency,--,float,The efficiency of methanol-powered ships in the conversion of methanol to meet shipping needs.
+shipping_oil_efficiency,--,float,The efficiency of oil-powered ships in the conversion of oil to meet shipping needs.
+aviation_demand_factor,--,float,The proportion of demand for aviation compared to today's
+HVC_demand_factor,--,float,The proportion of demand for high-value chemicals compared to today's
+time_dep_hp_cop,--,true or false,
+heat_pump_sink_T,°C,float,
+reduce_space_heat_exogenously,--,true or false,
+reduce_space_heat_exogenously_factor,--,Dictionary with planning horizons as keys.,
+retrofitting,,,
+-- retro_endogen,--,true or false,Add retrofitting as an endogenous system which co-optimise space heat savings.
+-- cost_factor,,,Weight costs for building renovation
+-- interest_rate,,,The interest rate for investment in building components
+-- annualise_cost,--,true or false,Annualise the investment costs of retrofitting
+-- tax_weighting,--,true or false,Weight the costs of retrofitting depending on taxes in countries
+-- construction_index,--,true or false,Weight the costs of retrofitting depending on labour/material costs per country
+tes,--,true or false,Add option for storing thermal energy in large water pits associated with district heating systems and individual thermal energy storage (TES)
+tes_tau,,,
+-- decentral,,,
+-- central,,,
+boilers,--,true or false,Add option for transforming electricity into heat using resistive heater
+oil_boilers,--,true or false,Add option for transforming oil into heat using boilers
+biomass_boiler,--,true or false,Add option for transforming biomass into heat using boilers
+chp,--,true or false,Add option for using Combined Heat and Power (CHP)
+micro_chp,--,true or false,Add option for using Combined Heat and Power (CHP) for decentral areas.
+solar_thermal,--,true or false,Add option for using solar to generate heat.
+solar_cf_correction,,,
+marginal_cost_storage,,,
+methanation,--,true or false,Add option for transforming hydrogen and CO2 into methane using methanation.
+helmeth,--,true or false,Add option for transforming power into gas using HELMETH (Integrated High-Temperature ELectrolysis and METHanation for Effective Power to Gas Conversion)
+coal_cc,--,true or false,Add option for coal CHPs with carbon capture
+dac,--,true or false,Add option for Direct Air Capture (DAC)
+co2_vent,--,true or false,Add option for vent out CO2 from storages to the atmosphere.
+allam_cycle,--,true or false,Add option to include `Allam cycle gas power plants <https://en.wikipedia.org/wiki/Allam_power_cycle>`_
+hydrogen_fuel_cell,--,true or false,Add option to include hydrogen fuel cell for re-electrification. Assuming OCGT technology costs
+hydrogen_turbine,--,true or false,Add option to include hydrogen turbine for re-electrification. Assuming OCGT technology costs
+SMR,--,true or false,Add option for transforming natural gas into hydrogen and CO2 using Steam Methane Reforming (SMR)
+regional_co2_sequestration_potential,,,
+-- enable,--,true or false,Add option for regionally-resolved geological carbon dioxide sequestration potentials based on `CO2StoP <https://setis.ec.europa.eu/european-co2-storage-database_en>`_.
+-- attribute,,,
+-- include_onshore,,true or false,Add options for including onshore sequestration potentials
+-- min_size,,float,Any sites with lower potential than this value will be excluded
+-- max_size,,float,The maximum sequestration potential for any one site.
+-- years_of_storage,,float,The years until potential exhausted at optimised annual rate
+co2_sequestration_potential,MtCO2/a,float,The potential of sequestering CO2 in Europe per year
+co2_sequestration_cost,EUR/tCO2,float,The cost of sequestering a ton of CO2
+co2_spatial,--,true or false,"„Add option to spatially resolve carrier representing stored carbon dioxide. This allows for more detailed modelling of CCUTS, e.g. regarding the capturing of industrial process emissions, usage as feedstock for electrofuels, transport of carbon dioxide, and geological sequestration sites.“"
+co2network,--,true or false,Add option for planning a new carbon dioxide network
+cc_fraction,,,The default fraction of CO2 captured with post-combustion capture
+hydrogen_underground_storage,--,true or false,Add options for storing hydrogen underground. Storage potential depends regionally.
+hydrogen_underground_storage_locations,,"„onshore, nearshore or offshore“","„The location where hydrogen underground storage can be located. Onshore, nearshore, offshore means it must be located more than 50 km away from the sea, within 50 km of the sea, or within the sea itself respectively.“"
+ammonia,--,"„true, false or regional“","Add ammonia as a carrrier. It can be either true (copperplated NH3) or ""regional"" (regionalised NH3 without network)"
+min_part_load_fischer_tropsch,,,
+min_part_load_methanolisation,,,
+use_fischer_tropsch_waste_heat,--,true or false,Add option for using waste heat of Fischer Tropsch in district heating networks
+use_fuel_cell_waste_heat,--,true or false,Add option for using waste heat of fuel cells in district heating networks
+use_electrolysis_waste_heat,--,true or false,Add option for using waste heat of electrolysis in district heating networks
+electricity_distribution_grid,--,true or false,Add a electricity distribution grid
+electricity_distribution_grid_cost_factor,,,Multiplies the investment cost of the electricity distribution grid in data/costs.csv
+electricity_grid_connection,--,true or false,Add the cost of electricity grid connection for onshore wind and solar
+H2_network,--,true or false,Add option for new hydrogen pipelines
+gas_network,--,true or false,"Add natural gas infrastructure, incl. LNG terminals, production and entry-points"
+H2_retrofit,--,true or false,Add option for retrofiting existing pipelines to transport hydrogen
+H2_retrofit_capacity_per_CH4,,,
+gas_distribution_grid,--,true or false,Add a gas distribution grid
+gas_distribution_grid_cost_factor,,,Multiplies the investment cost of the gas distribution grid in data/costs.csv
+biomass_spatial,--,true or false,Add option for resolving biomass demand regionally
+biomass_transport,--,true or false,Add option for transporting solid biomass between nodes
+conventional_generation,,,Add a more detailed description of conventional carriers. Any power generation requires the consumption of fuel from nodes representing that fuel.
+biomass_to_liquid,--,true or false,Add option for transforming solid biomass into liquid fuel with the same properties as oil
+biosng,--,true or false,Add option for transforming solid biomass into synthesis gas with the same properties as natural gas
diff --git a/doc/configuration.rst b/doc/configuration.rst
index 89b6f42f..a1e0379e 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -471,6 +471,11 @@ overwrite the existing values.
    :start-at: sector:
    :end-before: # docs
 
+.. csv-table::
+   :header-rows: 1
+   :widths: 25,7,22,30
+   :file: configtables/sector.csv
+
 .. _industry_cf:
 
 ``industry``
@@ -487,6 +492,11 @@ overwrite the existing values.
    :start-at: industry:
    :end-before: # docs
 
+.. csv-table::
+   :header-rows: 1
+   :widths: 25,7,22,30
+   :file: configtables/industry.csv
+
 .. _costs_cf:
 
 ``costs``

From febd2933742874924e316cb3e40b4c47b5ed5211 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 28 Jun 2023 08:05:16 +0000
Subject: [PATCH 135/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 config/config.default.yaml     | 36 ++++++++++++++++------------------
 doc/configtables/countries.csv |  2 +-
 doc/configtables/foresight.csv |  2 +-
 doc/configtables/sector.csv    |  2 +-
 doc/configtables/toplevel.csv  |  2 +-
 doc/configuration.rst          | 16 +++++++--------
 6 files changed, 29 insertions(+), 31 deletions(-)

diff --git a/config/config.default.yaml b/config/config.default.yaml
index dc04a925..a54309ef 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: CC0-1.0
 
-# docs in 
+# docs in
 version: 0.8.0
 tutorial: false
 
@@ -10,22 +10,22 @@ logging:
   level: INFO
   format: '%(levelname)s:%(name)s:%(message)s'
 
-# docs in 
-run: 
+# docs in
+run:
   name: ""
   disable_progressbar: false
   shared_resources: false
   shared_cutouts: true
 
-# docs in 
+# docs in
 foresight: overnight
 
-# docs in 
-# Wildcard docs in 
-scenario: 
+# docs in
+# Wildcard docs in
+scenario:
   simpl:
   - ''
-  ll:  
+  ll:
   - v1.5
   clusters:
   - 37
@@ -33,7 +33,7 @@ scenario:
   - 256
   - 512
   - 1024
-  opts: 
+  opts:
   - ''
   sector_opts:
   - Co2L0-3H-T-H-B-I-A-solar+p3-dist1
@@ -43,18 +43,16 @@ scenario:
   # - 2040
   - 2050
 
-# docs in 
-countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 
-            'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 
-            'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK']
+# docs in
+countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK']
 
-# docs in 
+# docs in
 snapshots:
   start: "2013-01-01"
   end: "2014-01-01"
   inclusive: 'left'
 
-# docs in 
+# docs in
 enable:
   prepare_links_p_nom: false
   retrieve_databundle: true
@@ -66,7 +64,7 @@ enable:
   retrieve_natura_raster: true
   custom_busmap: false
 
-# docs in 
+# docs in
 co2_budget:
   2020: 0.701
   2025: 0.524
@@ -76,7 +74,7 @@ co2_budget:
   2045: 0.032
   2050: 0.000
 
-# docs in 
+# docs in
 electricity:
   voltages: [220., 300., 380.]
   gaslimit: false
@@ -157,7 +155,7 @@ renewable:
     resource:
       method: wind
       turbine: Vestas_V112_3MW
-    capacity_per_sqkm: 3 
+    capacity_per_sqkm: 3
     # correction_factor: 0.93
     corine:
       grid_codes: [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32]
@@ -172,7 +170,7 @@ renewable:
     resource:
       method: wind
       turbine: NREL_ReferenceTurbine_5MW_offshore
-    capacity_per_sqkm: 2 
+    capacity_per_sqkm: 2
     correction_factor: 0.8855
     corine: [44, 255]
     natura: true
diff --git a/doc/configtables/countries.csv b/doc/configtables/countries.csv
index 63dd85b3..6a386416 100644
--- a/doc/configtables/countries.csv
+++ b/doc/configtables/countries.csv
@@ -1,2 +1,2 @@
  ,Unit,Values,Description
-countries,--,"Subset of {'AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'}","European countries defined by their `Two-letter country codes (ISO 3166-1) <https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2>`_ which should be included in the energy system model."
\ No newline at end of file
+countries,--,"Subset of {'AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'}","European countries defined by their `Two-letter country codes (ISO 3166-1) <https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2>`_ which should be included in the energy system model."
diff --git a/doc/configtables/foresight.csv b/doc/configtables/foresight.csv
index 07f930f2..a19ec139 100644
--- a/doc/configtables/foresight.csv
+++ b/doc/configtables/foresight.csv
@@ -1,2 +1,2 @@
 ,Unit,Values,Description
-foresight,string,"{overnight, myopic, perfect}","See :ref:`Foresight Options` for detail explanations."
\ No newline at end of file
+foresight,string,"{overnight, myopic, perfect}","See :ref:`Foresight Options` for detail explanations."
diff --git a/doc/configtables/sector.csv b/doc/configtables/sector.csv
index 27a69809..6a6fcd0d 100644
--- a/doc/configtables/sector.csv
+++ b/doc/configtables/sector.csv
@@ -4,7 +4,7 @@ district_heating,--,,`prepare_sector_network.py <https://github.com/PyPSA/pypsa-
 -- progress,--,Dictionary with planning horizons as keys.,Progress = 0 means today's district heating share. Progress = 1 means maximum fraction of urban demand is supplied by district heating
 -- district_heating_loss,--,float,
 cluster_heat_buses,--,true or false,cluster residential and service heat buses in `prepare_sector_network.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/prepare_sector_network.py>`_  to one to save memory.
-bev_dsm_restriction_value,--,float,Adding a stage of charge (SOC) limit  for battery electric vehicles (BEV) to manage its own energy demand (DSM). Located in `build_transport_demand.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/build_transport_demand.py>`_. 
+bev_dsm_restriction_value,--,float,Adding a stage of charge (SOC) limit  for battery electric vehicles (BEV) to manage its own energy demand (DSM). Located in `build_transport_demand.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/build_transport_demand.py>`_.
 bev_dsm_restriction_time,--,float,Time at which SOC of BEV has to be dsm_restriction_value
 transport_heating_deadband_upper,°C,float,"The minimum temperature in the vehicle. At lower temperatures, the energy required for heating in the vehicle increases."
 transport_heating_deadband_lower,°C,float,"The maximum temperature in the vehicle. At higher temperatures, the energy required for cooling in the vehicle increases."
diff --git a/doc/configtables/toplevel.csv b/doc/configtables/toplevel.csv
index 9496e165..dcf0f29a 100644
--- a/doc/configtables/toplevel.csv
+++ b/doc/configtables/toplevel.csv
@@ -3,4 +3,4 @@ version,--,0.x.x,"Version of PyPSA-Eur. Descriptive only."
 tutorial,bool,"{true, false}","Switch to retrieve the tutorial data set instead of the full data set."
 logging,,,
 -- level,--,"Any of {'INFO', 'WARNING', 'ERROR'}","Restrict console outputs to all infos, warning or errors only"
--- format,--,"","Custom format for log messages. See `LogRecord <https://docs.python.org/3/library/logging.html#logging.LogRecord>`_ attributes."
\ No newline at end of file
+-- format,--,"","Custom format for log messages. See `LogRecord <https://docs.python.org/3/library/logging.html#logging.LogRecord>`_ attributes."
diff --git a/doc/configuration.rst b/doc/configuration.rst
index a1e0379e..a5de4453 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -64,7 +64,7 @@ The ``run`` section is used for running and storing scenarios with different con
    :file: configtables/foresight.csv
 
 .. note::
-    if you use myopic or perfect foresight, The investment years in 
+    if you use myopic or perfect foresight, The investment years in
     :ref:`planning_horizons` in scenario has to be set.
 
 .. _scenario:
@@ -173,7 +173,7 @@ Switches for some rules and optional features.
    :file: configtables/co2_budget.csv
 
 .. note::
-    this parameter is over-ridden if ``CO2Lx`` or ``cb`` is set in 
+    this parameter is over-ridden if ``CO2Lx`` or ``cb`` is set in
     sector_opts.
 
 .. _electricity_cf:
@@ -272,7 +272,7 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia
    :file: configtables/offwind-dc.csv
 
 .. note::
-   both ``offwind-ac`` and ``offwind-dc`` have the same assumption on 
+   both ``offwind-ac`` and ``offwind-dc`` have the same assumption on
    ``capacity_per_sqkm`` and ``correction_factor``.
 
 ``solar``
@@ -512,8 +512,8 @@ overwrite the existing values.
    :widths: 25,7,22,30
    :file: configtables/costs.csv
 
-.. note:: 
-   ``rooftop_share:`` are based on the potentials, assuming  
+.. note::
+   ``rooftop_share:`` are based on the potentials, assuming
    (0.1 kW/m2 and 10 m2/person)
 
 .. _clustering_cf:
@@ -531,11 +531,11 @@ overwrite the existing values.
    :widths: 25,7,22,30
    :file: configtables/clustering.csv
 
-.. note:: 
-   ``feature:`` in ``simplify_network:`` 
+.. note::
+   ``feature:`` in ``simplify_network:``
    are only relevant if ``hac`` were chosen in ``algorithm``.
 
-.. tip:: 
+.. tip::
    use ``min`` in ``p_nom_max:`` for more `
    conservative assumptions.
 

From 2f3005e55512cee79639c491c6244a24fdbc3b89 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Thu, 29 Jun 2023 15:37:29 +0200
Subject: [PATCH 136/176] use pypsa.clustering.spatial namespace

---
 scripts/_helpers.py         | 2 +-
 scripts/cluster_network.py  | 6 +++---
 scripts/simplify_network.py | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/scripts/_helpers.py b/scripts/_helpers.py
index 512cb87a..0e1d2327 100644
--- a/scripts/_helpers.py
+++ b/scripts/_helpers.py
@@ -283,7 +283,7 @@ def get_aggregation_strategies(aggregation_strategies):
     # when custom values are specified in the config.
 
     import numpy as np
-    from pypsa.networkclustering import _make_consense
+    from pypsa.clustering.spatial import _make_consense
 
     bus_strategies = dict(country=_make_consense("Bus", "country"))
     bus_strategies.update(aggregation_strategies.get("buses", {}))
diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py
index 52685af2..14741f9e 100644
--- a/scripts/cluster_network.py
+++ b/scripts/cluster_network.py
@@ -89,7 +89,7 @@ Description
     **Is it possible to run the model without the** ``simplify_network`` **rule?**
 
         No, the network clustering methods in the PyPSA module
-        `pypsa.networkclustering <https://github.com/PyPSA/PyPSA/blob/master/pypsa/networkclustering.py>`_
+        `pypsa.clustering.spatial <https://github.com/PyPSA/PyPSA/blob/master/pypsa/clustering/spatial.py>`_
         do not work reliably with multiple voltage levels and transformers.
 
 .. tip::
@@ -134,7 +134,7 @@ import pyomo.environ as po
 import pypsa
 import seaborn as sns
 from _helpers import configure_logging, get_aggregation_strategies, update_p_nom_max
-from pypsa.networkclustering import (
+from pypsa.clustering.spatial import (
     busmap_by_greedy_modularity,
     busmap_by_hac,
     busmap_by_kmeans,
@@ -484,7 +484,7 @@ if __name__ == "__main__":
         # Fast-path if no clustering is necessary
         busmap = n.buses.index.to_series()
         linemap = n.lines.index.to_series()
-        clustering = pypsa.networkclustering.Clustering(
+        clustering = pypsa.clustering.spatial.Clustering(
             n, busmap, linemap, linemap, pd.Series(dtype="O")
         )
     else:
diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 79161760..b4c0829b 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -95,13 +95,13 @@ import scipy as sp
 from _helpers import configure_logging, get_aggregation_strategies, update_p_nom_max
 from add_electricity import load_costs
 from cluster_network import cluster_regions, clustering_for_n_clusters
-from pypsa.io import import_components_from_dataframe, import_series_from_dataframe
-from pypsa.networkclustering import (
+from pypsa.clustering.spatial import (
     aggregategenerators,
     aggregateoneport,
     busmap_by_stubs,
     get_clustering_from_busmap,
 )
+from pypsa.io import import_components_from_dataframe, import_series_from_dataframe
 from scipy.sparse.csgraph import connected_components, dijkstra
 
 logger = logging.getLogger(__name__)

From 8c7c586b0c841a538cd6cbd3aaff9d1df093e875 Mon Sep 17 00:00:00 2001
From: Fabian Hofmann <fab.hof@gmx.de>
Date: Thu, 29 Jun 2023 16:02:03 +0200
Subject: [PATCH 137/176] Update scripts/add_extra_components.py

---
 scripts/add_extra_components.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py
index 006a50a3..14d428e4 100644
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@@ -115,7 +115,6 @@ def attach_stores(n, costs, extendable_carriers):
             capital_cost=costs.at["hydrogen storage underground", "capital_cost"],
         )
 
-        n.madd("Carrier", ["H2 electrolysis", "H2 fuel cell"])
 
         n.madd(
             "Link",

From cf06204e4f8644bef42c5e3eb7d37d329f3b5abf Mon Sep 17 00:00:00 2001
From: Fabian Hofmann <fab.hof@gmx.de>
Date: Thu, 29 Jun 2023 16:02:09 +0200
Subject: [PATCH 138/176] Update scripts/add_electricity.py

---
 scripts/add_electricity.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 8a8f8d00..f5847023 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -167,7 +167,7 @@ def sanitize_carriers(n, config):
     if colors.isna().any():
         missing_i = list(colors.index[colors.isna()])
         logger.warning(f"tech_colors for carriers {missing_i} not defined in config.")
-    n.carriers["color"] = colors
+    n.carriers["color"] = n.carriers.color.where(n.carriers.color != "", colors)
 
 
 def add_co2_emissions(n, costs, carriers):

From a9b0d349f24484aa8a885732eef943c140ad8222 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 29 Jun 2023 14:02:19 +0000
Subject: [PATCH 139/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/add_extra_components.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py
index 14d428e4..e00e1e5f 100644
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@@ -115,7 +115,6 @@ def attach_stores(n, costs, extendable_carriers):
             capital_cost=costs.at["hydrogen storage underground", "capital_cost"],
         )
 
-
         n.madd(
             "Link",
             h2_buses_i + " Electrolysis",

From 3f946dd83d9cf4fd4eb8c64381c7020d115ccb81 Mon Sep 17 00:00:00 2001
From: Fabian Hofmann <fab.hof@gmx.de>
Date: Thu, 29 Jun 2023 16:02:31 +0200
Subject: [PATCH 140/176] Update scripts/add_electricity.py

---
 scripts/add_electricity.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index f5847023..e6c8c01b 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -162,7 +162,7 @@ def sanitize_carriers(n, config):
         .reindex(carrier_i)
         .fillna(carrier_i.to_series().str.title())
     )
-    n.carriers["nice_name"] = nice_names
+    n.carriers["nice_name"] = n.carriers.nice_name.where(n.carriers.nice_name != "", nice_names)
     colors = pd.Series(config["plotting"]["tech_colors"]).reindex(carrier_i)
     if colors.isna().any():
         missing_i = list(colors.index[colors.isna()])

From d84c7ebf6acb0c7d012f7fe3aa40825e47e02fc3 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 29 Jun 2023 14:02:54 +0000
Subject: [PATCH 141/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/add_electricity.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index e6c8c01b..4b53fe99 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -162,7 +162,9 @@ def sanitize_carriers(n, config):
         .reindex(carrier_i)
         .fillna(carrier_i.to_series().str.title())
     )
-    n.carriers["nice_name"] = n.carriers.nice_name.where(n.carriers.nice_name != "", nice_names)
+    n.carriers["nice_name"] = n.carriers.nice_name.where(
+        n.carriers.nice_name != "", nice_names
+    )
     colors = pd.Series(config["plotting"]["tech_colors"]).reindex(carrier_i)
     if colors.isna().any():
         missing_i = list(colors.index[colors.isna()])

From 0dd1933057f3a8849114cba122f50881e791358c Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Thu, 29 Jun 2023 16:21:45 +0200
Subject: [PATCH 142/176] solve_network: fix argument for linopy groupby

---
 scripts/solve_network.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index d80e38a0..58f4db72 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -235,8 +235,7 @@ def add_CCL_constraints(n, config):
     p_nom = n.model["Generator-p_nom"]
 
     gens = n.generators.query("p_nom_extendable").rename_axis(index="Generator-ext")
-    grouper = [gens.bus.map(n.buses.country), gens.carrier]
-    grouper = xr.DataArray(pd.MultiIndex.from_arrays(grouper), dims=["Generator-ext"])
+    grouper = pd.concat([gens.bus.map(n.buses.country), gens.carrier])
     lhs = p_nom.groupby(grouper).sum().rename(bus="country")
 
     minimum = xr.DataArray(agg_p_nom_minmax["min"].dropna()).rename(dim_0="group")

From 00ec36a12e43be64ef9001fbdbdb98996f77e899 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Thu, 29 Jun 2023 16:38:56 +0200
Subject: [PATCH 143/176] add_electricity: filter out nan geocoords in kdtree

---
 scripts/add_electricity.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 594988bd..ecd3b70e 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -644,7 +644,7 @@ def attach_OPSD_renewables(n, tech_map):
         buses = n.buses.loc[gens.bus.unique()]
         gens_per_bus = gens.groupby("bus").p_nom.count()
 
-        caps = map_country_bus(df.query("Fueltype == @fueltype"), buses)
+        caps = map_country_bus(df.query("Fueltype == @fueltype and lat == lat"), buses)
         caps = caps.groupby(["bus"]).Capacity.sum()
         caps = caps / gens_per_bus.reindex(caps.index, fill_value=1)
 

From 9e143345bddb840f25da84e7c0f334b92b9434f2 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Thu, 29 Jun 2023 19:24:38 +0200
Subject: [PATCH 144/176] follow up on #686

---
 scripts/build_powerplants.py        | 1 +
 scripts/build_renewable_profiles.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py
index 7f396e1e..f25f111b 100755
--- a/scripts/build_powerplants.py
+++ b/scripts/build_powerplants.py
@@ -149,6 +149,7 @@ if __name__ == "__main__":
         logging.warning(f"No powerplants known in: {', '.join(countries_wo_ppl)}")
 
     substations = n.buses.query("substation_lv")
+    ppl = ppl.dropna(subset=["lat", "lon"])
     ppl = map_country_bus(ppl, substations)
 
     bus_null_b = ppl["bus"].isnull()
diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py
index aa23be3a..864d54dc 100644
--- a/scripts/build_renewable_profiles.py
+++ b/scripts/build_renewable_profiles.py
@@ -204,6 +204,7 @@ if __name__ == "__main__":
 
     nprocesses = int(snakemake.threads)
     noprogress = snakemake.config["run"].get("disable_progressbar", True)
+    noprogress = noprogress or not snakemake.config["atlite"]["show_progress"]
     params = snakemake.params.renewable[snakemake.wildcards.technology]
     resource = params["resource"]  # pv panel params / wind turbine params
     correction_factor = params.get("correction_factor", 1.0)

From 701231fdca3adb465744b46132c9fe57a6fd8346 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Fri, 30 Jun 2023 11:29:46 +0200
Subject: [PATCH 145/176] build_renewable_profile: fix execution for one thread

---
 rules/build_electricity.smk         | 3 +++
 rules/retrieve.smk                  | 2 --
 scripts/build_renewable_profiles.py | 8 ++++----
 scripts/retrieve_databundle.py      | 2 +-
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index 44ebf404..c7ef86a2 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -59,6 +59,9 @@ rule build_powerplants:
 
 
 rule base_network:
+    params:
+        countries=config["countries"],
+        snapshots=config["snapshots"],
     input:
         eg_buses="data/entsoegridkit/buses.csv",
         eg_lines="data/entsoegridkit/lines.csv",
diff --git a/rules/retrieve.smk b/rules/retrieve.smk
index 4bfbd6c6..0a96406a 100644
--- a/rules/retrieve.smk
+++ b/rules/retrieve.smk
@@ -19,8 +19,6 @@ if config["enable"].get("retrieve_databundle", True):
         datafiles.extend(["natura/Natura2000_end2015.shp", "GEBCO_2014_2D.nc"])
 
     rule retrieve_databundle:
-        params:
-            tutorial=config["tutorial"],
         output:
             expand("data/bundle/{file}", file=datafiles),
         log:
diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py
index 864d54dc..88a43f53 100644
--- a/scripts/build_renewable_profiles.py
+++ b/scripts/build_renewable_profiles.py
@@ -188,7 +188,7 @@ import geopandas as gpd
 import numpy as np
 import xarray as xr
 from _helpers import configure_logging
-from dask.distributed import Client, LocalCluster
+from dask.distributed import Client
 from pypsa.geo import haversine
 from shapely.geometry import LineString
 
@@ -218,8 +218,7 @@ if __name__ == "__main__":
         logger.info(f"correction_factor is set as {correction_factor}")
 
     if nprocesses > 1:
-        # cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1)
-        client = Client(n_workers=2, threads_per_worker=2, memory_limit="1GB")
+        client = Client(n_workers=nprocesses, threads_per_worker=1)
     else:
         client = None
 
@@ -293,7 +292,8 @@ if __name__ == "__main__":
 
     potential = capacity_per_sqkm * availability.sum("bus") * area
     func = getattr(cutout, resource.pop("method"))
-    resource["dask_kwargs"] = {"scheduler": client}
+    if client is not None:
+        resource["dask_kwargs"] = {"scheduler": client}
     capacity_factor = correction_factor * func(capacity_factor=True, **resource)
     layout = capacity_factor * area * capacity_per_sqkm
     profile, capacities = func(
diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py
index c7053bed..75d8519e 100644
--- a/scripts/retrieve_databundle.py
+++ b/scripts/retrieve_databundle.py
@@ -53,7 +53,7 @@ if __name__ == "__main__":
         snakemake
     )  # TODO Make logging compatible with progressbar (see PR #102)
 
-    if snakemake.params.tutorial:
+    if snakemake.config["tutorial"]:
         url = "https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz"
     else:
         url = "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz"

From 16dcad01760aaa5cdba2a07522bbdac247245481 Mon Sep 17 00:00:00 2001
From: euronion <42553970+euronion@users.noreply.github.com>
Date: Fri, 30 Jun 2023 13:25:20 +0200
Subject: [PATCH 146/176] Prevent attempt to re-add existing carriers.

---
 doc/release_notes.rst      |  2 ++
 scripts/add_electricity.py | 19 ++++++++++++-------
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index ddd1483e..727675c2 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -16,6 +16,8 @@ Upcoming Release
 
 * Bugfix: Correct typo in the CPLEX solver configuration in ``config.default.yaml``.
 
+* Bugfix: Error in ``add_electricity`` where carriers were added multiple times to the network, resulting in a non-unique carriers error.
+
 * Renamed script file from PyPSA-EUR ``build_load_data`` to ``build_electricity_demand`` and ``retrieve_load_data`` to ``retrieve_electricity_demand``.
 
 * Fix docs readthedocs built
diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 88860790..0e528644 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -122,6 +122,13 @@ def calculate_annuity(n, r):
     else:
         return 1 / n
 
+def add_missing_carriers(n, carriers):
+    """
+    Function to add missing carriers to the network without raising errors.
+    """
+    missing_carriers = set(carriers) - set(n.carriers.index)
+    if len(missing_carriers) > 0:
+        n.madd("Carrier", missing_carriers)
 
 def sanitize_carriers(n, config):
     """
@@ -152,9 +159,7 @@ def sanitize_carriers(n, config):
 
     for c in n.iterate_components():
         if "carrier" in c.df:
-            missing_carrier = set(c.df.carrier.unique()) - set(n.carriers.index) - {""}
-            if len(missing_carrier):
-                n.madd("Carrier", missing_carrier)
+            add_missing_carriers(n, c.df)
 
     carrier_i = n.carriers.index
     nice_names = (
@@ -354,7 +359,7 @@ def update_transmission_costs(n, costs, length_factor=1.0):
 def attach_wind_and_solar(
     n, costs, input_profiles, carriers, extendable_carriers, line_length_factor=1
 ):
-    n.madd("Carrier", carriers)
+    add_missing_carriers(n, carriers)
 
     for car in carriers:
         if car == "hydro":
@@ -416,7 +421,7 @@ def attach_conventional_generators(
     conventional_inputs,
 ):
     carriers = list(set(conventional_carriers) | set(extendable_carriers["Generator"]))
-    n.madd("Carrier", carriers)
+    add_missing_carriers(n, carriers)
     add_co2_emissions(n, costs, carriers)
 
     ppl = (
@@ -473,7 +478,7 @@ def attach_conventional_generators(
 
 
 def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **params):
-    n.madd("Carrier", carriers)
+    add_missing_carriers(n, carriers)
     add_co2_emissions(n, costs, carriers)
 
     ppl = (
@@ -606,7 +611,7 @@ def attach_extendable_generators(n, costs, ppl, carriers):
     logger.warning(
         "The function `attach_extendable_generators` is deprecated in v0.5.0."
     )
-    n.madd("Carrier", carriers)
+    add_missing_carriers(n, carriers)
     add_co2_emissions(n, costs, carriers)
 
     for tech in carriers:

From e4651ce711d2bd1f53736e3ec0abf77c4b137c04 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 30 Jun 2023 11:28:54 +0000
Subject: [PATCH 147/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/add_electricity.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 0e528644..2f6251f0 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -122,6 +122,7 @@ def calculate_annuity(n, r):
     else:
         return 1 / n
 
+
 def add_missing_carriers(n, carriers):
     """
     Function to add missing carriers to the network without raising errors.
@@ -130,6 +131,7 @@ def add_missing_carriers(n, carriers):
     if len(missing_carriers) > 0:
         n.madd("Carrier", missing_carriers)
 
+
 def sanitize_carriers(n, config):
     """
     Sanitize the carrier information in a PyPSA Network object.

From 9138901e5d35467331a1956dde856fff6c3e1d49 Mon Sep 17 00:00:00 2001
From: virio-andreyana <arsyan.virio@gmail.com>
Date: Mon, 3 Jul 2023 03:37:41 +0200
Subject: [PATCH 148/176] add table for biomass, energy, existing capacities,
 solar thermal, plotting (need suggestion)

---
 doc/configtables/biomass.csv             |  7 ++
 doc/configtables/energy.csv              |  5 ++
 doc/configtables/existing_capacities.csv |  5 ++
 doc/configtables/plotting.csv            | 29 +++++---
 doc/configtables/sector.csv              | 92 ++++++++++++------------
 doc/configtables/solar-thermal.csv       |  5 ++
 doc/configuration.rst                    | 56 ++++++++++-----
 7 files changed, 127 insertions(+), 72 deletions(-)
 create mode 100644 doc/configtables/biomass.csv
 create mode 100644 doc/configtables/energy.csv
 create mode 100644 doc/configtables/existing_capacities.csv
 create mode 100644 doc/configtables/solar-thermal.csv

diff --git a/doc/configtables/biomass.csv b/doc/configtables/biomass.csv
new file mode 100644
index 00000000..f5b4841f
--- /dev/null
+++ b/doc/configtables/biomass.csv
@@ -0,0 +1,7 @@
+,Unit,Values,Description
+year ,--,"{2010, 2020, 2030, 2040, 2050}",Year for which to retrieve biomass potential according to the assumptions of the `JRC ENSPRESO <https://data.jrc.ec.europa.eu/dataset/74ed5a04-7d74-4807-9eab-b94774309d9f>`_ .
+scenario ,--,"{""ENS_Low"", ""ENS_Med"", ""ENS_High""}",Scenario for which to retrieve biomass potential. The scenario definition can be seen in `ENSPRESO_BIOMASS <https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx>`_
+classes ,,,
+-- solid biomass,--,Array of biomass comodity,The comodity that are included as solid biomass
+-- not included,--,Array of biomass comodity,The comodity that are not included as a biomass potential
+-- biogas,--,Array of biomass comodity,The comodity that are included as biogas
diff --git a/doc/configtables/energy.csv b/doc/configtables/energy.csv
new file mode 100644
index 00000000..640a9bc3
--- /dev/null
+++ b/doc/configtables/energy.csv
@@ -0,0 +1,5 @@
+,Unit,Values,Description
+energy_totals_year ,--,"{1990,1995,2000,2005,2010,2011,…} ",The year for the sector energy use. The year must be avaliable in the Eurostat report
+base_emissions_year ,--,YYYY; e.g. ‚1990‘ ,'The base year for the sector emissions. See `European Environment Agency (EEA) <https://www.eea.europa.eu/data-and-maps/data/national-emissions-reported-to-the-unfccc-and-to-the-eu-greenhouse-gas-monitoring-mechanism-16>`_ downloaded 201228 (modified by EEA last on 201221)‘
+eurostat_report_year ,--,"{2016,2017,2018}","'The year in which the Eurostat report year are extracted. 2016 includes Bosnia and Herzegovina, 2017 does not‘"
+emissions ,--,"{CO2, All greenhouse gases - (CO2 equivalent)}",Specify which sectoral emissions are taken into account. Data derived from EEA.
diff --git a/doc/configtables/existing_capacities.csv b/doc/configtables/existing_capacities.csv
new file mode 100644
index 00000000..6b53d5f1
--- /dev/null
+++ b/doc/configtables/existing_capacities.csv
@@ -0,0 +1,5 @@
+,Unit,Values,Description
+grouping_years_power ,--,An array of years,Intervals to group existing capacities for power
+grouping_years_heat ,--,An array of years below 2020,Intervals to group existing capacities for heat
+threshold_capacity ,MW,float,Capacities generators and links of below threshold are removed during add_existing_capacities
+conventional_carriers ,--,"Any subset of {uranium, coal, lignite, oil} ",List of conventional power plants to include in the sectoral network
diff --git a/doc/configtables/plotting.csv b/doc/configtables/plotting.csv
index bea345ca..82bf7af6 100644
--- a/doc/configtables/plotting.csv
+++ b/doc/configtables/plotting.csv
@@ -1,10 +1,19 @@
-,Unit,Values,Description
-map,,,
--- boundaries,°,"[x1,x2,y1,y2]","Boundaries of the map plots in degrees latitude (y) and longitude (x)"
-costs_max,bn Euro,float,"Upper y-axis limit in cost bar plots."
-costs_threshold,bn Euro,float,"Threshold below which technologies will not be shown in cost bar plots."
-energy_max,TWh,float,"Upper y-axis limit in energy bar plots."
-energy_min,TWh,float,"Lower y-axis limit in energy bar plots."
-energy_threshold,TWh,float,"Threshold below which technologies will not be shown in energy bar plots."
-tech_colors,--,"carrier -> HEX colour code","Mapping from network ``carrier`` to a colour (`HEX colour code <https://en.wikipedia.org/wiki/Web_colors#Hex_triplet>`_)."
-nice_names,--,"str -> str","Mapping from network ``carrier`` to a more readable name."
+,Unit,Values,Description
+map,,,
+-- boundaries,°,"[x1,x2,y1,y2]",Boundaries of the map plots in degrees latitude (y) and longitude (x)
+costs_max,bn Euro,float,Upper y-axis limit in cost bar plots.
+costs_threshold,bn Euro,float,Threshold below which technologies will not be shown in cost bar plots.
+energy_max,TWh,float,Upper y-axis limit in energy bar plots.
+energy_min,TWh,float,Lower y-axis limit in energy bar plots.
+energy_threshold,TWh,float,Threshold below which technologies will not be shown in energy bar plots.
+tech_colors,--,carrier -> HEX colour code,Mapping from network ``carrier`` to a colour (`HEX colour code <https://en.wikipedia.org/wiki/Web_colors#Hex_triplet>`_).
+vre_techs ,--,subset of carriers,Value not in use
+renewable_storage_techs ,--,subset of carriers,Value not in use
+conv_techs ,--,subset of carriers,Value in use with unworthy significance. Used as the definition of conventional carrier in solve_network
+storage_techs ,--,subset of carriers,Value not in use
+load_carriers ,--,subset of carriers,Value not in use
+AC_carriers ,--,subset of carriers,Value not in use
+link_carriers ,--,subset of carriers,Value not in use
+heat_links ,--,subset of carriers,Value not in use
+heat_generators ,--,subset of carriers,Value not in use
+nice_names,--,str -> str,Mapping from network ``carrier`` to a more readable name.
diff --git a/doc/configtables/sector.csv b/doc/configtables/sector.csv
index 6a6fcd0d..7a6f3c14 100644
--- a/doc/configtables/sector.csv
+++ b/doc/configtables/sector.csv
@@ -3,7 +3,7 @@ district_heating,--,,`prepare_sector_network.py <https://github.com/PyPSA/pypsa-
 -- potential,--,float,maximum fraction of urban demand which can be supplied by district heating increase of today's district heating demand to potential maximum district heating share
 -- progress,--,Dictionary with planning horizons as keys.,Progress = 0 means today's district heating share. Progress = 1 means maximum fraction of urban demand is supplied by district heating
 -- district_heating_loss,--,float,
-cluster_heat_buses,--,true or false,cluster residential and service heat buses in `prepare_sector_network.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/prepare_sector_network.py>`_  to one to save memory.
+cluster_heat_buses,--,"{true, false}",cluster residential and service heat buses in `prepare_sector_network.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/prepare_sector_network.py>`_  to one to save memory.
 bev_dsm_restriction_value,--,float,Adding a stage of charge (SOC) limit  for battery electric vehicles (BEV) to manage its own energy demand (DSM). Located in `build_transport_demand.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/build_transport_demand.py>`_.
 bev_dsm_restriction_time,--,float,Time at which SOC of BEV has to be dsm_restriction_value
 transport_heating_deadband_upper,°C,float,"The minimum temperature in the vehicle. At lower temperatures, the energy required for heating in the vehicle increases."
@@ -12,7 +12,7 @@ ICE_lower_degree_factor,--,float,Percentage increase in energy demand in interna
 ICE_upper_degree_factor,--,float,Percentage increase in energy demand in internal combustion engine (ICE) for each degree difference between the hot environment and the maximum temperature.
 EV_lower_degree_factor,--,float,Percentage increase in energy demand in electric vehicles (EV) for each degree difference between the cold environment and the minimum temperature.
 EV_upper_degree_factor,--,float,Percentage increase in energy demand in electric vehicles (EV) for each degree difference between the hot environment and the maximum temperature.
-bev_dsm,--,true or false,Add the option for battery electric vehicles (BEV) to manage its own energy demand (DSM)
+bev_dsm,--,"{true, false}",Add the option for battery electric vehicles (BEV) to manage its own energy demand (DSM)
 bev_availability,--,float,The percentage for battery electric vehicles (BEV) that are able to do demand side management (DSM)
 bev_energy,--,float,The average size of battery electric vehicles (BEV) in MWh
 bev_charge_efficiency,--,float,Battery electric vehicles (BEV) charge and discharge efficiency
@@ -20,7 +20,7 @@ bev_plug_to_wheel_efficiency,km/kWh,float,The distance battery electric vehicles
 bev_charge_rate,MWh,float,The power consumption for one electric vehicle (EV) in MWh. Value derived from 3-phase charger with 11 kW.
 bev_avail_max,--,float,The maximum percentage plugged-in availability for passenger electric vehicles.
 bev_avail_mean,--,float,The average percentage plugged-in availability for passenger electric vehicles.
-v2g,--,true or false,Allows feed-in to grid from EV battery
+v2g,--,"{true, false}",Allows feed-in to grid from EV battery
 land_transport_fuel_cell_share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses fuel cells in a given year
 land_transport_electric_share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses electric vehicles (EV) in a given year
 land_transport_ice_share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses internal combustion engines (ICE) in a given year
@@ -33,7 +33,7 @@ agriculture_machinery_electric_efficiency,--,float,The efficiency of oil-powered
 MWh_MeOH_per_MWh_H2,LHV,float,The energy amount of the produced methanol per energy amount of hydrogen. source: DECHEMA (2017): Low carbon energy and feedstock for the European chemical industry page 64.
 MWh_MeOH_per_tCO2,LHV,float,The energy amount of the produced methanol per ton of CO2
 MWh_MeOH_per_MWh_e,LHV,float,The energy amount of the produced methanol per energy amount of electricity
-shipping_hydrogen_liquefaction,--,true or false,Consider whether to include liquefaction costs for shipping H2 demand.
+shipping_hydrogen_liquefaction,--,"{true, false}",Consider whether to include liquefaction costs for shipping H2 demand.
 shipping_hydrogen_share,--,Dictionary with planning horizons as keys.,The share of ships powered by hydrogen in a given year
 shipping_methanol_share,--,Dictionary with planning horizons as keys.,The share of ships powered by methanol in a given year
 shipping_oil_share,--,Dictionary with planning horizons as keys.,The share of ships powered by oil in a given year
@@ -41,69 +41,69 @@ shipping_methanol_efficiency,--,float,The efficiency of methanol-powered ships i
 shipping_oil_efficiency,--,float,The efficiency of oil-powered ships in the conversion of oil to meet shipping needs.
 aviation_demand_factor,--,float,The proportion of demand for aviation compared to today's
 HVC_demand_factor,--,float,The proportion of demand for high-value chemicals compared to today's
-time_dep_hp_cop,--,true or false,
+time_dep_hp_cop,--,"{true, false}",
 heat_pump_sink_T,°C,float,
-reduce_space_heat_exogenously,--,true or false,
+reduce_space_heat_exogenously,--,"{true, false}",
 reduce_space_heat_exogenously_factor,--,Dictionary with planning horizons as keys.,
 retrofitting,,,
--- retro_endogen,--,true or false,Add retrofitting as an endogenous system which co-optimise space heat savings.
+-- retro_endogen,--,"{true, false}",Add retrofitting as an endogenous system which co-optimise space heat savings.
 -- cost_factor,,,Weight costs for building renovation
 -- interest_rate,,,The interest rate for investment in building components
--- annualise_cost,--,true or false,Annualise the investment costs of retrofitting
--- tax_weighting,--,true or false,Weight the costs of retrofitting depending on taxes in countries
--- construction_index,--,true or false,Weight the costs of retrofitting depending on labour/material costs per country
-tes,--,true or false,Add option for storing thermal energy in large water pits associated with district heating systems and individual thermal energy storage (TES)
+-- annualise_cost,--,"{true, false}",Annualise the investment costs of retrofitting
+-- tax_weighting,--,"{true, false}",Weight the costs of retrofitting depending on taxes in countries
+-- construction_index,--,"{true, false}",Weight the costs of retrofitting depending on labour/material costs per country
+tes,--,"{true, false}",Add option for storing thermal energy in large water pits associated with district heating systems and individual thermal energy storage (TES)
 tes_tau,,,
 -- decentral,,,
 -- central,,,
-boilers,--,true or false,Add option for transforming electricity into heat using resistive heater
-oil_boilers,--,true or false,Add option for transforming oil into heat using boilers
-biomass_boiler,--,true or false,Add option for transforming biomass into heat using boilers
-chp,--,true or false,Add option for using Combined Heat and Power (CHP)
-micro_chp,--,true or false,Add option for using Combined Heat and Power (CHP) for decentral areas.
-solar_thermal,--,true or false,Add option for using solar to generate heat.
+boilers,--,"{true, false}",Add option for transforming electricity into heat using resistive heater
+oil_boilers,--,"{true, false}",Add option for transforming oil into heat using boilers
+biomass_boiler,--,"{true, false}",Add option for transforming biomass into heat using boilers
+chp,--,"{true, false}",Add option for using Combined Heat and Power (CHP)
+micro_chp,--,"{true, false}",Add option for using Combined Heat and Power (CHP) for decentral areas.
+solar_thermal,--,"{true, false}",Add option for using solar to generate heat.
 solar_cf_correction,,,
 marginal_cost_storage,,,
-methanation,--,true or false,Add option for transforming hydrogen and CO2 into methane using methanation.
-helmeth,--,true or false,Add option for transforming power into gas using HELMETH (Integrated High-Temperature ELectrolysis and METHanation for Effective Power to Gas Conversion)
-coal_cc,--,true or false,Add option for coal CHPs with carbon capture
-dac,--,true or false,Add option for Direct Air Capture (DAC)
-co2_vent,--,true or false,Add option for vent out CO2 from storages to the atmosphere.
-allam_cycle,--,true or false,Add option to include `Allam cycle gas power plants <https://en.wikipedia.org/wiki/Allam_power_cycle>`_
-hydrogen_fuel_cell,--,true or false,Add option to include hydrogen fuel cell for re-electrification. Assuming OCGT technology costs
-hydrogen_turbine,--,true or false,Add option to include hydrogen turbine for re-electrification. Assuming OCGT technology costs
-SMR,--,true or false,Add option for transforming natural gas into hydrogen and CO2 using Steam Methane Reforming (SMR)
+methanation,--,"{true, false}",Add option for transforming hydrogen and CO2 into methane using methanation.
+helmeth,--,"{true, false}",Add option for transforming power into gas using HELMETH (Integrated High-Temperature ELectrolysis and METHanation for Effective Power to Gas Conversion)
+coal_cc,--,"{true, false}",Add option for coal CHPs with carbon capture
+dac,--,"{true, false}",Add option for Direct Air Capture (DAC)
+co2_vent,--,"{true, false}",Add option for vent out CO2 from storages to the atmosphere.
+allam_cycle,--,"{true, false}",Add option to include `Allam cycle gas power plants <https://en.wikipedia.org/wiki/Allam_power_cycle>`_
+hydrogen_fuel_cell,--,"{true, false}",Add option to include hydrogen fuel cell for re-electrification. Assuming OCGT technology costs
+hydrogen_turbine,--,"{true, false}",Add option to include hydrogen turbine for re-electrification. Assuming OCGT technology costs
+SMR,--,"{true, false}",Add option for transforming natural gas into hydrogen and CO2 using Steam Methane Reforming (SMR)
 regional_co2_sequestration_potential,,,
--- enable,--,true or false,Add option for regionally-resolved geological carbon dioxide sequestration potentials based on `CO2StoP <https://setis.ec.europa.eu/european-co2-storage-database_en>`_.
+-- enable,--,"{true, false}",Add option for regionally-resolved geological carbon dioxide sequestration potentials based on `CO2StoP <https://setis.ec.europa.eu/european-co2-storage-database_en>`_.
 -- attribute,,,
--- include_onshore,,true or false,Add options for including onshore sequestration potentials
+-- include_onshore,,"{true, false}",Add options for including onshore sequestration potentials
 -- min_size,,float,Any sites with lower potential than this value will be excluded
 -- max_size,,float,The maximum sequestration potential for any one site.
 -- years_of_storage,,float,The years until potential exhausted at optimised annual rate
 co2_sequestration_potential,MtCO2/a,float,The potential of sequestering CO2 in Europe per year
 co2_sequestration_cost,EUR/tCO2,float,The cost of sequestering a ton of CO2
-co2_spatial,--,true or false,"„Add option to spatially resolve carrier representing stored carbon dioxide. This allows for more detailed modelling of CCUTS, e.g. regarding the capturing of industrial process emissions, usage as feedstock for electrofuels, transport of carbon dioxide, and geological sequestration sites.“"
-co2network,--,true or false,Add option for planning a new carbon dioxide network
+co2_spatial,--,"{true, false}","„Add option to spatially resolve carrier representing stored carbon dioxide. This allows for more detailed modelling of CCUTS, e.g. regarding the capturing of industrial process emissions, usage as feedstock for electrofuels, transport of carbon dioxide, and geological sequestration sites.“"
+co2network,--,"{true, false}",Add option for planning a new carbon dioxide network
 cc_fraction,,,The default fraction of CO2 captured with post-combustion capture
-hydrogen_underground_storage,--,true or false,Add options for storing hydrogen underground. Storage potential depends regionally.
-hydrogen_underground_storage_locations,,"„onshore, nearshore or offshore“","„The location where hydrogen underground storage can be located. Onshore, nearshore, offshore means it must be located more than 50 km away from the sea, within 50 km of the sea, or within the sea itself respectively.“"
-ammonia,--,"„true, false or regional“","Add ammonia as a carrrier. It can be either true (copperplated NH3) or ""regional"" (regionalised NH3 without network)"
+hydrogen_underground_storage,--,"{true, false}",Add options for storing hydrogen underground. Storage potential depends regionally.
+hydrogen_underground_storage_locations,,"{onshore, nearshore, offshore}","„The location where hydrogen underground storage can be located. Onshore, nearshore, offshore means it must be located more than 50 km away from the sea, within 50 km of the sea, or within the sea itself respectively.“"
+ammonia,--,"{true, false, regional}","Add ammonia as a carrrier. It can be either true (copperplated NH3) or ""regional"" (regionalised NH3 without network)"
 min_part_load_fischer_tropsch,,,
 min_part_load_methanolisation,,,
-use_fischer_tropsch_waste_heat,--,true or false,Add option for using waste heat of Fischer Tropsch in district heating networks
-use_fuel_cell_waste_heat,--,true or false,Add option for using waste heat of fuel cells in district heating networks
-use_electrolysis_waste_heat,--,true or false,Add option for using waste heat of electrolysis in district heating networks
-electricity_distribution_grid,--,true or false,Add a electricity distribution grid
+use_fischer_tropsch_waste_heat,--,"{true, false}",Add option for using waste heat of Fischer Tropsch in district heating networks
+use_fuel_cell_waste_heat,--,"{true, false}",Add option for using waste heat of fuel cells in district heating networks
+use_electrolysis_waste_heat,--,"{true, false}",Add option for using waste heat of electrolysis in district heating networks
+electricity_distribution_grid,--,"{true, false}",Add a electricity distribution grid
 electricity_distribution_grid_cost_factor,,,Multiplies the investment cost of the electricity distribution grid in data/costs.csv
-electricity_grid_connection,--,true or false,Add the cost of electricity grid connection for onshore wind and solar
-H2_network,--,true or false,Add option for new hydrogen pipelines
-gas_network,--,true or false,"Add natural gas infrastructure, incl. LNG terminals, production and entry-points"
-H2_retrofit,--,true or false,Add option for retrofiting existing pipelines to transport hydrogen
+electricity_grid_connection,--,"{true, false}",Add the cost of electricity grid connection for onshore wind and solar
+H2_network,--,"{true, false}",Add option for new hydrogen pipelines
+gas_network,--,"{true, false}","Add natural gas infrastructure, incl. LNG terminals, production and entry-points"
+H2_retrofit,--,"{true, false}",Add option for retrofiting existing pipelines to transport hydrogen
 H2_retrofit_capacity_per_CH4,,,
-gas_distribution_grid,--,true or false,Add a gas distribution grid
+gas_distribution_grid,--,"{true, false}",Add a gas distribution grid
 gas_distribution_grid_cost_factor,,,Multiplies the investment cost of the gas distribution grid in data/costs.csv
-biomass_spatial,--,true or false,Add option for resolving biomass demand regionally
-biomass_transport,--,true or false,Add option for transporting solid biomass between nodes
+biomass_spatial,--,"{true, false}",Add option for resolving biomass demand regionally
+biomass_transport,--,"{true, false}",Add option for transporting solid biomass between nodes
 conventional_generation,,,Add a more detailed description of conventional carriers. Any power generation requires the consumption of fuel from nodes representing that fuel.
-biomass_to_liquid,--,true or false,Add option for transforming solid biomass into liquid fuel with the same properties as oil
-biosng,--,true or false,Add option for transforming solid biomass into synthesis gas with the same properties as natural gas
+biomass_to_liquid,--,"{true, false}",Add option for transforming solid biomass into liquid fuel with the same properties as oil
+biosng,--,"{true, false}",Add option for transforming solid biomass into synthesis gas with the same properties as natural gas
diff --git a/doc/configtables/solar-thermal.csv b/doc/configtables/solar-thermal.csv
new file mode 100644
index 00000000..2daf99ae
--- /dev/null
+++ b/doc/configtables/solar-thermal.csv
@@ -0,0 +1,5 @@
+,Unit,Values,Description
+clearsky_model ,--,"{‘simple’, ‘enhanced’}",Type of clearsky model for diffuse irradiation 
+orientation ,--,"{units of degrees, ‘latitude_optimal’}",Panel orientation with slope and azimuth 
+-- azimuth,float,units of degrees,The angle between the north and the sun with panels on the local horizon
+-- slope,float,units of degrees,The angle between the ground and the panels
diff --git a/doc/configuration.rst b/doc/configuration.rst
index a5de4453..cd926b6a 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -398,14 +398,15 @@ overwrite the existing values.
 .. note::
    Only used for sector-coupling studies.
 
-.. warning::
-   More comprehensive documentation for this segment will be released soon.
-
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: energy:
    :end-before: # docs
 
+.. csv-table::
+   :header-rows: 1
+   :widths: 25,7,22,30
+   :file: configtables/energy.csv
 
 .. _biomass_cf:
 
@@ -415,14 +416,36 @@ overwrite the existing values.
 .. note::
    Only used for sector-coupling studies.
 
-.. warning::
-   More comprehensive documentation for this segment will be released soon.
-
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: biomass:
    :end-before: # docs
 
+.. csv-table::
+   :header-rows: 1
+   :widths: 25,7,22,30
+   :file: configtables/biomass.csv
+
+The list of available biomass is given by the category in `ENSPRESO_BIOMASS <https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx>`_, namely:
+
+- Agricultural waste
+- Manure solid, liquid
+- Residues from landscape care
+- Bioethanol barley, wheat, grain maize, oats, other cereals and rye
+- Sugar from sugar beet
+- Miscanthus, switchgrass, RCG
+- Willow
+- Poplar
+- Sunflower, soya seed 
+- Rape seed
+- Fuelwood residues
+- FuelwoodRW
+- C&P_RW
+- Secondary Forestry residues - woodchips
+- Sawdust
+- Municipal waste
+- Sludge
+
 .. _solar_thermal_cf:
 
 ``solar_thermal``
@@ -431,30 +454,34 @@ overwrite the existing values.
 .. note::
    Only used for sector-coupling studies.
 
-.. warning::
-   More comprehensive documentation for this segment will be released soon.
-
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: solar_thermal:
    :end-before: # docs
 
+.. csv-table::
+   :header-rows: 1
+   :widths: 25,7,22,30
+   :file: configtables/solar-thermal.csv
+
 .. _existing_capacities_cf:
 
 ``existing_capacities``
 =======================
 
 .. note::
-   Only used for sector-coupling studies.
-
-.. warning::
-   More comprehensive documentation for this segment will be released soon.
+   Only used for sector-coupling studies. The value for grouping years are only used in myopic scenarios.
 
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: existing_capacities:
    :end-before: # docs
 
+.. csv-table::
+   :header-rows: 1
+   :widths: 25,7,22,30
+   :file: configtables/existing_capacities.csv
+
 .. _sector_cf:
 
 ``sector``
@@ -463,9 +490,6 @@ overwrite the existing values.
 .. note::
    Only used for sector-coupling studies.
 
-.. warning::
-   More comprehensive documentation for this segment will be released soon.
-
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: sector:

From ac264282dab1fd2d5d930f8aa591564f91c1af18 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 3 Jul 2023 01:38:02 +0000
Subject: [PATCH 149/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 doc/configtables/solar-thermal.csv | 4 ++--
 doc/configuration.rst              | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/configtables/solar-thermal.csv b/doc/configtables/solar-thermal.csv
index 2daf99ae..7cc8dfc7 100644
--- a/doc/configtables/solar-thermal.csv
+++ b/doc/configtables/solar-thermal.csv
@@ -1,5 +1,5 @@
 ,Unit,Values,Description
-clearsky_model ,--,"{‘simple’, ‘enhanced’}",Type of clearsky model for diffuse irradiation 
-orientation ,--,"{units of degrees, ‘latitude_optimal’}",Panel orientation with slope and azimuth 
+clearsky_model ,--,"{‘simple’, ‘enhanced’}",Type of clearsky model for diffuse irradiation
+orientation ,--,"{units of degrees, ‘latitude_optimal’}",Panel orientation with slope and azimuth
 -- azimuth,float,units of degrees,The angle between the north and the sun with panels on the local horizon
 -- slope,float,units of degrees,The angle between the ground and the panels
diff --git a/doc/configuration.rst b/doc/configuration.rst
index cd926b6a..66af161e 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -436,7 +436,7 @@ The list of available biomass is given by the category in `ENSPRESO_BIOMASS <htt
 - Miscanthus, switchgrass, RCG
 - Willow
 - Poplar
-- Sunflower, soya seed 
+- Sunflower, soya seed
 - Rape seed
 - Fuelwood residues
 - FuelwoodRW

From d751f4694d713505be004af3da9a8634e1328f76 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Mon, 3 Jul 2023 18:58:15 +0200
Subject: [PATCH 150/176] Update scripts/prepare_network.py

---
 scripts/prepare_network.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index 1c286818..7b7f77f9 100755
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -349,8 +349,8 @@ if __name__ == "__main__":
         n,
         s_nom_max_set=snakemake.params.lines.get("s_nom_max", np.inf),
         p_nom_max_set=snakemake.params.links.get("p_nom_max", np.inf),
-        s_nom_max_set=snakemake.params.lines.get("max_extension", np.inf),
-        p_nom_max_set=snakemake.params.links.get("max_extension", np.inf),
+        s_nom_max_ext=snakemake.params.lines.get("max_extension", np.inf),
+        p_nom_max_ext=snakemake.params.links.get("max_extension", np.inf),
     )
 
     if "ATK" in opts:

From 55add4c7857faa7d3a4d5677995ea7a57e8e7a46 Mon Sep 17 00:00:00 2001
From: virio-andreyana <arsyan.virio@gmail.com>
Date: Wed, 5 Jul 2023 01:51:50 +0200
Subject: [PATCH 151/176] finalized config documentation

---
 config/config.default.yaml          | 236 +++++++++++++---------------
 doc/configtables/.~lock.sector.csv# |   1 +
 doc/configtables/electricity.csv    |  58 +++----
 doc/configtables/industry.csv       |  40 ++---
 doc/configtables/sector.csv         |  69 ++++----
 doc/configuration.rst               |   8 +-
 6 files changed, 197 insertions(+), 215 deletions(-)
 create mode 100644 doc/configtables/.~lock.sector.csv#

diff --git a/config/config.default.yaml b/config/config.default.yaml
index 8a1948aa..f3b69e4a 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: CC0-1.0
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#top-level-configuration
 version: 0.8.0
 tutorial: false
 
@@ -10,18 +10,18 @@ logging:
   level: INFO
   format: '%(levelname)s:%(name)s:%(message)s'
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#run
 run:
   name: ""
   disable_progressbar: false
   shared_resources: false
   shared_cutouts: true
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight
 foresight: overnight
 
-# docs in
-# Wildcard docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#scenario
+# Wildcard docs in https://pypsa-eur.readthedocs.io/en/latest/wildcards.html
 scenario:
   simpl:
   - ''
@@ -43,16 +43,16 @@ scenario:
   # - 2040
   - 2050
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#countries
 countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK']
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#snapshots
 snapshots:
   start: "2013-01-01"
   end: "2014-01-01"
   inclusive: 'left'
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#enable
 enable:
   prepare_links_p_nom: false
   retrieve_databundle: true
@@ -64,7 +64,7 @@ enable:
   retrieve_natura_raster: true
   custom_busmap: false
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#co2-budget
 co2_budget:
   2020: 0.701
   2025: 0.524
@@ -74,7 +74,7 @@ co2_budget:
   2045: 0.032
   2050: 0.000
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#electricity
 electricity:
   voltages: [220., 300., 380.]
   gaslimit: false
@@ -99,7 +99,6 @@ electricity:
     Link: [] # H2 pipeline
 
   powerplants_filter: (DateOut >= 2022 or DateOut != DateOut)
-  # use pandas query strings here, e.g. Country in ['Germany']
   custom_powerplants: false
 
   conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass]
@@ -107,25 +106,19 @@ electricity:
 
   estimate_renewable_capacities:
     enable: true
-    # Add capacities from OPSD data
     from_opsd: true
-    # Renewable capacities are based on existing capacities reported by IRENA
     year: 2020
-    # Artificially limit maximum capacities to factor * (IRENA capacities),
-    # i.e. 110% of <years>'s capacities => expansion_limit: 1.1
-    # false: Use estimated renewable potentials determine by the workflow
     expansion_limit: false
     technology_mapping:
-      # Wind is the Fueltype in powerplantmatching, onwind, offwind-{ac,dc} the carrier in PyPSA-Eur
       Offshore: [offwind-ac, offwind-dc]
       Onshore: [onwind]
       PV: [solar]
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#atlite
 atlite:
   default_cutout: europe-2013-era5
   nprocesses: 4
-  show_progress: false # false saves time
+  show_progress: false
   cutouts:
     # use 'base' to determine geographical bounds and time span from config
     # base:
@@ -148,7 +141,7 @@ atlite:
       sarah_dir:
       features: [influx, temperature]
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#renewable
 renewable:
   onwind:
     cutout: europe-2013-era5
@@ -217,12 +210,12 @@ renewable:
     hydro_max_hours: "energy_capacity_totals_by_country" # one of energy_capacity_totals_by_country, estimate_by_large_installations or a float
     clip_min_inflow: 1.0
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#conventional
 conventional:
   nuclear:
     p_max_pu: "data/nuclear_p_max_pu.csv" # float of file name
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#lines
 lines:
   types:
     220.: "Al/St 240/40 2-bundle 220.0"
@@ -233,20 +226,20 @@ lines:
   length_factor: 1.25
   under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#links
 links:
   p_max_pu: 1.0
   p_nom_max: .inf
   include_tyndp: true
   under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#transformers
 transformers:
   x: 0.1
   s_nom: 2000.
   type: ''
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#load
 load:
   power_statistics: true
   interpolate_limit: 3
@@ -254,6 +247,7 @@ load:
   manual_adjustments: true # false
   scaling_factor: 1.0
 
+# docs
 # TODO: PyPSA-Eur merge issue in prepare_sector_network.py
 # regulate what components with which carriers are kept from PyPSA-Eur;
 # some technologies are removed because they are implemented differently
@@ -275,14 +269,14 @@ pypsa_eur:
   - hydro
   Store: []
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#energy
 energy:
   energy_totals_year: 2011
   base_emissions_year: 1990
   eurostat_report_year: 2016
-  emissions: CO2 # "CO2" or "All greenhouse gases - (CO2 equivalent)"
+  emissions: CO2
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#biomass
 biomass:
   year: 2030
   scenario: ENS_Med
@@ -308,15 +302,14 @@ biomass:
     - Manure solid, liquid
     - Sludge
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#solar-thermal
 solar_thermal:
   clearsky_model: simple  # should be "simple" or "enhanced"?
   orientation:
     slope: 45.
     azimuth: 180.
 
-# docs under construction in
-# only relevant for foresight = myopic or perfect
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#existing-capacities
 existing_capacities:
   grouping_years_power: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030]
   grouping_years_heat: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019] # these should not extend 2020
@@ -327,37 +320,34 @@ existing_capacities:
   - oil
   - uranium
 
-# docs under construction in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#sector
 sector:
   district_heating:
-    potential: 0.6  # maximum fraction of urban demand which can be supplied by district heating
-     # increase of today's district heating demand to potential maximum district heating share
-     # progress = 0 means today's district heating share, progress = 1 means maximum fraction of urban demand is supplied by district heating
+    potential: 0.6 
     progress:
       2020: 0.0
       2030: 0.3
       2040: 0.6
       2050: 1.0
     district_heating_loss: 0.15
-  cluster_heat_buses: false # cluster residential and service heat buses to one to save memory
-  bev_dsm_restriction_value: 0.75  #Set to 0 for no restriction on BEV DSM
-  bev_dsm_restriction_time: 7  #Time at which SOC of BEV has to be dsm_restriction_value
+  cluster_heat_buses: false
+  bev_dsm_restriction_value: 0.75
+  bev_dsm_restriction_time: 7
   transport_heating_deadband_upper: 20.
   transport_heating_deadband_lower: 15.
-  ICE_lower_degree_factor: 0.375  #in per cent increase in fuel consumption per degree above deadband
+  ICE_lower_degree_factor: 0.375
   ICE_upper_degree_factor: 1.6
   EV_lower_degree_factor: 0.98
   EV_upper_degree_factor: 0.63
-  bev_dsm: true #turns on EV battery
-  bev_availability: 0.5  #How many cars do smart charging
-  bev_energy: 0.05  #average battery size in MWh
-  bev_charge_efficiency: 0.9  #BEV (dis-)charging efficiency
-  bev_plug_to_wheel_efficiency: 0.2 #kWh/km from EPA https://www.fueleconomy.gov/feg/ for Tesla Model S
-  bev_charge_rate: 0.011 #3-phase charger with 11 kW
+  bev_dsm: true
+  bev_availability: 0.5
+  bev_energy: 0.05
+  bev_charge_efficiency: 0.9
+  bev_plug_to_wheel_efficiency: 0.2
+  bev_charge_rate: 0.011
   bev_avail_max: 0.95
   bev_avail_mean: 0.8
-  v2g: true #allows feed-in to grid from EV battery
-  #what is not EV or FCEV is oil-fuelled ICE
+  v2g: true
   land_transport_fuel_cell_share:
     2020: 0
     2030: 0.05
@@ -377,12 +367,12 @@ sector:
   transport_internal_combustion_efficiency: 0.3
   agriculture_machinery_electric_share: 0
   agriculture_machinery_oil_share: 1
-  agriculture_machinery_fuel_efficiency: 0.7 # fuel oil per use
-  agriculture_machinery_electric_efficiency: 0.3 # electricity per use
-  MWh_MeOH_per_MWh_H2: 0.8787 # in LHV, source: DECHEMA (2017): Low carbon energy and feedstock for the European chemical industry , pg. 64.
-  MWh_MeOH_per_tCO2: 4.0321 # in LHV, source: DECHEMA (2017): Low carbon energy and feedstock for the European chemical industry , pg. 64.
-  MWh_MeOH_per_MWh_e: 3.6907 # in LHV, source: DECHEMA (2017): Low carbon energy and feedstock for the European chemical industry , pg. 64.
-  shipping_hydrogen_liquefaction: false # whether to consider liquefaction costs for shipping H2 demands
+  agriculture_machinery_fuel_efficiency: 0.7
+  agriculture_machinery_electric_efficiency: 0.3
+  MWh_MeOH_per_MWh_H2: 0.8787
+  MWh_MeOH_per_tCO2: 4.0321
+  MWh_MeOH_per_MWh_e: 3.6907
+  shipping_hydrogen_liquefaction: false
   shipping_hydrogen_share:
     2020: 0
     2030: 0
@@ -398,18 +388,14 @@ sector:
     2030: 0.7
     2040: 0.3
     2050: 0
-  shipping_methanol_efficiency: 0.46 # 10-15% higher https://www.iea-amf.org/app/webroot/files/file/Annex%20Reports/AMF_Annex_56.pdf, https://users.ugent.be/~lsileghe/documents/extended_abstract.pdf
-  shipping_oil_efficiency: 0.40 #For conversion of fuel oil to propulsion in 2011
-  aviation_demand_factor: 1. # relative aviation demand compared to today
-  HVC_demand_factor: 1. # relative HVC demand compared to today
-  time_dep_hp_cop: true #time dependent heat pump coefficient of performance
-  heat_pump_sink_T: 55. # Celsius, based on DTU / large area radiators; used in build_cop_profiles.py
-   # conservatively high to cover hot water and space heating in poorly-insulated buildings
-  reduce_space_heat_exogenously: true  # reduces space heat demand by a given factor (applied before losses in DH)
-  # this can represent e.g. building renovation, building demolition, or if
-  # the factor is negative: increasing floor area, increased thermal comfort, population growth
-  reduce_space_heat_exogenously_factor:  # per unit reduction in space heat demand
-  # the default factors are determined by the LTS scenario from http://tool.european-calculator.eu/app/buildings/building-types-area/?levers=1ddd4444421213bdbbbddd44444ffffff11f411111221111211l212221
+  shipping_methanol_efficiency: 0.46
+  shipping_oil_efficiency: 0.40
+  aviation_demand_factor: 1.
+  HVC_demand_factor: 1.
+  time_dep_hp_cop: true
+  heat_pump_sink_T: 55.
+  reduce_space_heat_exogenously: true
+  reduce_space_heat_exogenously_factor:
     2020: 0.10  # this results in a space heat demand reduction of 10%
     2025: 0.09  # first heat demand increases compared to 2020 because of larger floor area per capita
     2030: 0.09
@@ -417,15 +403,15 @@ sector:
     2040: 0.16
     2045: 0.21
     2050: 0.29
-  retrofitting:   # co-optimises building renovation to reduce space heat demand
-    retro_endogen: false  # co-optimise space heat savings
-    cost_factor: 1.0   # weight costs for building renovation
-    interest_rate: 0.04  # for investment in building components
-    annualise_cost: true  # annualise the investment costs
-    tax_weighting: false   # weight costs depending on taxes in countries
-    construction_index: true   # weight costs depending on labour/material costs per country
+  retrofitting:
+    retro_endogen: false
+    cost_factor: 1.0
+    interest_rate: 0.04
+    annualise_cost: true
+    tax_weighting: false
+    construction_index: true
   tes: true
-  tes_tau: # 180 day time constant for centralised, 3 day for decentralised
+  tes_tau:
     decentral: 3
     central: 180
   boilers: true
@@ -446,51 +432,48 @@ sector:
   hydrogen_turbine: false
   SMR: true
   regional_co2_sequestration_potential:
-    enable: false  # enable regionally resolved geological co2 storage potential
+    enable: false
     attribute: 'conservative estimate Mt'
-    include_onshore: false  # include onshore sequestration potentials
-    min_size: 3 # Gt, sites with lower potential will be excluded
-    max_size: 25 # Gt, max sequestration potential for any one site, TODO research suitable value
-    years_of_storage: 25 # years until potential exhausted at optimised annual rate
-  co2_sequestration_potential: 200  #MtCO2/a sequestration potential for Europe
-  co2_sequestration_cost: 10   #EUR/tCO2 for sequestration of CO2
+    include_onshore: false 
+    min_size: 3
+    max_size: 25
+    years_of_storage: 25
+  co2_sequestration_potential: 200
+  co2_sequestration_cost: 10
   co2_spatial: false
   co2network: false
-  cc_fraction: 0.9  # default fraction of CO2 captured with post-combustion capture
+  cc_fraction: 0.9
   hydrogen_underground_storage: true
   hydrogen_underground_storage_locations:
     # - onshore  # more than 50 km from sea
   - nearshore    # within 50 km of sea
     # - offshore
-  ammonia: false # can be false (no NH3 carrier), true (copperplated NH3), "regional" (regionalised NH3 without network)
-  min_part_load_fischer_tropsch: 0.9 # p_min_pu
-  min_part_load_methanolisation: 0.5 # p_min_pu
+  ammonia: false
+  min_part_load_fischer_tropsch: 0.9
+  min_part_load_methanolisation: 0.5
   use_fischer_tropsch_waste_heat: true
   use_fuel_cell_waste_heat: true
   use_electrolysis_waste_heat: false
   electricity_distribution_grid: true
-  electricity_distribution_grid_cost_factor: 1.0  #multiplies cost in data/costs.csv
-  electricity_grid_connection: true  # only applies to onshore wind and utility PV
+  electricity_distribution_grid_cost_factor: 1.0 
+  electricity_grid_connection: true
   H2_network: true
   gas_network: false
-  H2_retrofit: false  # if set to True existing gas pipes can be retrofitted to H2 pipes
-  # according to hydrogen backbone strategy (April, 2020) p.15
-  # https://gasforclimate2050.eu/wp-content/uploads/2020/07/2020_European-Hydrogen-Backbone_Report.pdf
-  # 60% of original natural gas capacity could be used in cost-optimal case as H2 capacity
-  H2_retrofit_capacity_per_CH4: 0.6  # ratio for H2 capacity per original CH4 capacity of retrofitted pipelines
-  gas_network_connectivity_upgrade: 1 # https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation
+  H2_retrofit: false
+  H2_retrofit_capacity_per_CH4: 0.6 
+  gas_network_connectivity_upgrade: 1
   gas_distribution_grid: true
-  gas_distribution_grid_cost_factor: 1.0  #multiplies cost in data/costs.csv
-  biomass_spatial: false  # regionally resolve biomass (e.g. potentials)
-  biomass_transport: false  # allow transport of solid biomass between nodes
-  conventional_generation: # generator : carrier
+  gas_distribution_grid_cost_factor: 1.0
+  biomass_spatial: false
+  biomass_transport: false
+  conventional_generation:
     OCGT: gas
   biomass_to_liquid: false
   biosng: false
 
-# docs under construction in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#industry
 industry:
-  St_primary_fraction:  # fraction of steel produced via primary route versus secondary route (scrap+EAF); today fraction is 0.6
+  St_primary_fraction:
     2020: 0.6
     2025: 0.55
     2030: 0.5
@@ -498,7 +481,7 @@ industry:
     2040: 0.4
     2045: 0.35
     2050: 0.3
-  DRI_fraction:  # fraction of the primary route converted to DRI + EAF
+  DRI_fraction:
     2020: 0
     2025: 0
     2030: 0.05
@@ -506,9 +489,9 @@ industry:
     2040: 0.4
     2045: 0.7
     2050: 1
-  H2_DRI: 1.7   #H2 consumption in Direct Reduced Iron (DRI),  MWh_H2,LHV/ton_Steel from 51kgH2/tSt in Vogl et al (2018) doi:10.1016/j.jclepro.2018.08.279
-  elec_DRI: 0.322   #electricity consumption in Direct Reduced Iron (DRI) shaft, MWh/tSt HYBRIT brochure https://ssabwebsitecdn.azureedge.net/-/media/hybrit/files/hybrit_brochure.pdf
-  Al_primary_fraction:  # fraction of aluminium produced via the primary route versus scrap; today fraction is 0.4
+  H2_DRI: 1.7 
+  elec_DRI: 0.322
+  Al_primary_fraction:
     2020: 0.4
     2025: 0.375
     2030: 0.35
@@ -516,33 +499,30 @@ industry:
     2040: 0.3
     2045: 0.25
     2050: 0.2
-  MWh_NH3_per_tNH3: 5.166 # LHV
-  MWh_CH4_per_tNH3_SMR: 10.8 # 2012's demand from https://ec.europa.eu/docsroom/documents/4165/attachments/1/translations/en/renditions/pdf
-  MWh_elec_per_tNH3_SMR: 0.7 # same source, assuming 94-6% split methane-elec of total energy demand 11.5 MWh/tNH3
-  MWh_H2_per_tNH3_electrolysis: 6.5 # from https://doi.org/10.1016/j.joule.2018.04.017, around 0.197 tH2/tHN3 (>3/17 since some H2 lost and used for energy)
-  MWh_elec_per_tNH3_electrolysis: 1.17 # from https://doi.org/10.1016/j.joule.2018.04.017 Table 13 (air separation and HB)
+  MWh_NH3_per_tNH3: 5.166
+  MWh_CH4_per_tNH3_SMR: 10.8
+  MWh_elec_per_tNH3_SMR: 0.7
+  MWh_H2_per_tNH3_electrolysis: 6.5
+  MWh_elec_per_tNH3_electrolysis: 1.17
   MWh_NH3_per_MWh_H2_cracker: 1.46 # https://github.com/euronion/trace/blob/44a5ff8401762edbef80eff9cfe5a47c8d3c8be4/data/efficiencies.csv
-  NH3_process_emissions: 24.5 # in MtCO2/a from SMR for H2 production for NH3 from UNFCCC for 2015 for EU28
-  petrochemical_process_emissions: 25.5 # in MtCO2/a for petrochemical and other from UNFCCC for 2015 for EU28
-  HVC_primary_fraction: 1. # fraction of today's HVC produced via primary route
-  HVC_mechanical_recycling_fraction: 0. # fraction of today's HVC produced via mechanical recycling
-  HVC_chemical_recycling_fraction: 0. # fraction of today's HVC produced via chemical recycling
-  HVC_production_today: 52. # MtHVC/a from DECHEMA (2017), Figure 16, page 107; includes ethylene, propylene and BTX
-  MWh_elec_per_tHVC_mechanical_recycling: 0.547 # from SI of https://doi.org/10.1016/j.resconrec.2020.105010, Table S5, for HDPE, PP, PS, PET. LDPE would be 0.756.
-  MWh_elec_per_tHVC_chemical_recycling: 6.9 # Material Economics (2019), page 125; based on pyrolysis and electric steam cracking
-  chlorine_production_today: 9.58 # MtCl/a from DECHEMA (2017), Table 7, page 43
-  MWh_elec_per_tCl: 3.6 # DECHEMA (2017), Table 6, page 43
-  MWh_H2_per_tCl: -0.9372  # DECHEMA (2017), page 43; negative since hydrogen produced in chloralkali process
-  methanol_production_today: 1.5 # MtMeOH/a from DECHEMA (2017), page 62
-  MWh_elec_per_tMeOH: 0.167 # DECHEMA (2017), Table 14, page 65
-  MWh_CH4_per_tMeOH: 10.25 # DECHEMA (2017), Table 14, page 65
+  NH3_process_emissions: 24.5
+  petrochemical_process_emissions: 25.5
+  HVC_primary_fraction: 1.
+  HVC_mechanical_recycling_fraction: 0.
+  HVC_chemical_recycling_fraction: 0.
+  HVC_production_today: 52.
+  MWh_elec_per_tHVC_mechanical_recycling: 0.547
+  MWh_elec_per_tHVC_chemical_recycling: 6.9
+  chlorine_production_today: 9.58
+  MWh_elec_per_tCl: 3.6
+  MWh_H2_per_tCl: -0.9372
+  methanol_production_today: 1.5
+  MWh_elec_per_tMeOH: 0.167
+  MWh_CH4_per_tMeOH: 10.25
   hotmaps_locate_missing: false
   reference_year: 2015
-  # references:
-  # DECHEMA (2017): https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf
-  # Material Economics (2019): https://materialeconomics.com/latest-updates/industrial-transformation-2050
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#costs
 costs:
   year: 2030
   version: v0.5.0
@@ -572,7 +552,7 @@ costs:
   emission_prices:
     co2: 0.
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#clustering
 clustering:
   simplify_network:
     to_substations: false
@@ -596,7 +576,7 @@ clustering:
       ramp_limit_down: max
       efficiency: mean
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#solving
 solving:
   #tmpdir: "path/to/tmp"
   options:
@@ -670,7 +650,7 @@ solving:
 
   mem: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2
 
-# docs in
+# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#plotting
 plotting:
   map:
     boundaries: [-11, 30, 34, 71]
diff --git a/doc/configtables/.~lock.sector.csv# b/doc/configtables/.~lock.sector.csv#
new file mode 100644
index 00000000..b614fb4e
--- /dev/null
+++ b/doc/configtables/.~lock.sector.csv#
@@ -0,0 +1 @@
+,DESKTOP-2RNCH2B/kunde,,05.07.2023 01:09,file:///C:/Users/kunde/AppData/Roaming/LibreOffice/4;
\ No newline at end of file
diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv
index 9cf23ebf..02dcbdde 100644
--- a/doc/configtables/electricity.csv
+++ b/doc/configtables/electricity.csv
@@ -1,29 +1,29 @@
-,Unit,Values,Description
-voltages,kV,"Any subset of {220., 300., 380.}",Voltage levels to consider
-gaslimit,MWhth,"float or false",Global gas usage limit
-co2limit,:math:`t_{CO_2-eq}/a`,float,Cap on total annual system carbon dioxide emissions
-co2base,:math:`t_{CO_2-eq}/a`,float,Reference value of total annual system carbon dioxide emissions if relative emission reduction target is specified in ``{opts}`` wildcard.
-agg_p_nom_limits,file,path,Reference to ``.csv`` file specifying per carrier generator nominal capacity constraints for individual countries if ``'CCL'`` is in ``{opts}`` wildcard. Defaults to ``data/agg_p_nom_minmax.csv``.
-operational_reserve,,,"Settings for reserve requirements following like `GenX <https://genxproject.github.io/GenX/dev/core/#Reserves>`_"
--- activate,bool,"true or false","Whether to take operational reserve requirements into account during optimisation"
--- epsilon_load,--,float,share of total load
--- epsilon_vres,--,float,share of total renewable supply
--- contingency,MW,float,fixed reserve capacity
-max_hours,,,
--- battery,h,float,Maximum state of charge capacity of the battery in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation <https://pypsa.readthedocs.io/en/latest/components.html#storage-unit>`_.
--- H2,h,float,Maximum state of charge capacity of the hydrogen storage in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation <https://pypsa.readthedocs.io/en/latest/components.html#storage-unit>`_.
-extendable_carriers,,,
--- Generator,--,"Any extendable carrier","Defines existing or non-existing conventional and renewable power plants to be extendable during the optimization. Conventional generators can only be built/expanded where already existent today. If a listed conventional carrier is not included in the ``conventional_carriers`` list, the lower limit of the capacity expansion is set to 0."
--- StorageUnit,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity.
--- Store,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity.
--- Link,--,Any subset of {'H2 pipeline'},Adds extendable links (H2 pipelines only) at every connection where there are lines or HVDC links without capacity limits and with zero initial capacity. Hydrogen pipelines require hydrogen storage to be modelled as ``Store``.
-powerplants_filter,--,"use `pandas.query <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.query.html>`_ strings here, e.g. Country not in ['Germany']",Filter query for the default powerplant database.
-custom_powerplants,--,"use `pandas.query <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.query.html>`_ strings here, e.g. Country in ['Germany']",Filter query for the custom powerplant database.
-conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to include in the model from ``resources/powerplants.csv``. If an included carrier is also listed in `extendable_carriers`, the capacity is taken as a lower bound."
-renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, hydro}",List of renewable generators to include in the model.
-estimate_renewable_capacities,,,
--- enable,,bool,"Activate routine to estimate renewable capacities"
--- from_opsd,--,bool,"Add capacities from OPSD data"
--- year,--,bool,"Renewable capacities are based on existing capacities reported by IRENA for the specified year"
--- expansion_limit,--,float or false,"Artificially limit maximum capacities to factor * (IRENA capacities), i.e. 110% of <years>'s capacities => expansion_limit: 1.1 false: Use estimated renewable potentials determine by the workflow"
--- technology_mapping,,,"Mapping between powerplantmatching and PyPSA-Eur technology names"
+,Unit,Values,Description
+voltages,kV,"Any subset of {220., 300., 380.}",Voltage levels to consider
+gaslimit,MWhth,float or false,Global gas usage limit
+co2limit,:math:`t_{CO_2-eq}/a`,float,Cap on total annual system carbon dioxide emissions
+co2base,:math:`t_{CO_2-eq}/a`,float,Reference value of total annual system carbon dioxide emissions if relative emission reduction target is specified in ``{opts}`` wildcard.
+agg_p_nom_limits,file,path,Reference to ``.csv`` file specifying per carrier generator nominal capacity constraints for individual countries if ``'CCL'`` is in ``{opts}`` wildcard. Defaults to ``data/agg_p_nom_minmax.csv``.
+operational_reserve,,,Settings for reserve requirements following like `GenX <https://genxproject.github.io/GenX/dev/core/#Reserves>`_
+-- activate,bool,true or false,Whether to take operational reserve requirements into account during optimisation
+-- epsilon_load,--,float,share of total load
+-- epsilon_vres,--,float,share of total renewable supply
+-- contingency,MW,float,fixed reserve capacity
+max_hours,,,
+-- battery,h,float,Maximum state of charge capacity of the battery in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation <https://pypsa.readthedocs.io/en/latest/components.html#storage-unit>`_.
+-- H2,h,float,Maximum state of charge capacity of the hydrogen storage in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation <https://pypsa.readthedocs.io/en/latest/components.html#storage-unit>`_.
+extendable_carriers,,,
+-- Generator,--,Any extendable carrier,"Defines existing or non-existing conventional and renewable power plants to be extendable during the optimization. Conventional generators can only be built/expanded where already existent today. If a listed conventional carrier is not included in the ``conventional_carriers`` list, the lower limit of the capacity expansion is set to 0."
+-- StorageUnit,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity.
+-- Store,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity.
+-- Link,--,Any subset of {'H2 pipeline'},Adds extendable links (H2 pipelines only) at every connection where there are lines or HVDC links without capacity limits and with zero initial capacity. Hydrogen pipelines require hydrogen storage to be modelled as ``Store``.
+powerplants_filter,--,"use `pandas.query <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.query.html>`_ strings here, e.g. Country not in ['Germany']",Filter query for the default powerplant database.
+custom_powerplants,--,"use `pandas.query <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.query.html>`_ strings here, e.g. Country in ['Germany']",Filter query for the custom powerplant database.
+conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to include in the model from ``resources/powerplants.csv``. If an included carrier is also listed in `extendable_carriers`, the capacity is taken as a lower bound."
+renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, hydro}",List of renewable generators to include in the model.
+estimate_renewable_capacities,,,
+-- enable,,bool,Activate routine to estimate renewable capacities
+-- from_opsd,--,bool,Add capacities from OPSD data
+-- year,--,bool,Renewable capacities are based on existing capacities reported by IRENA for the specified year
+-- expansion_limit,--,float or false,"Artificially limit maximum IRENA capacities to a factor.  For example, an ``expansion_limit: 1.1`` means 110% of capacities . If false are chosen, the estimated renewable potentials determine by the workflow are used."
+-- technology_mapping,,,Mapping between powerplantmatching and PyPSA-Eur technology names
diff --git a/doc/configtables/industry.csv b/doc/configtables/industry.csv
index 6d503498..248ed4b5 100644
--- a/doc/configtables/industry.csv
+++ b/doc/configtables/industry.csv
@@ -1,28 +1,28 @@
 ,Unit,Values,Description
 St_primary_fraction,--,Dictionary with planning horizons as keys.,The fraction of steel produced via primary route versus secondary route (scrap+EAF). Current fraction is 0.6
 DRI_fraction,--,Dictionary with planning horizons as keys.,The fraction of the primary route converted to DRI + EAF
-H2_DRI,--,float,The hydrogen consumption in Direct Reduced Iron (DRI)  Mwh_H2 LHV/ton_Steel from 51kgH2/tSt in Vogl et al (2018) doi:10.1016/j.jclepro.2018.08.279
-elec_DRI,--,float,The electricity consumed in Direct Reduced Iron (DRI) shaft. MWh/tSt HYBRIT brochure https://ssabwebsitecdn.azureedge.net/-/media/hybrit/files/hybrit_brochure.pdf
-Al_primary_fraction,--,Dictionary with planning horizons as keys.,The fraction of aluminium produced via the primary route versus scrap.  Current fraction is 0.4
+H2_DRI,--,float,The hydrogen consumption in Direct Reduced Iron (DRI)  Mwh_H2 LHV/ton_Steel from 51kgH2/tSt in `Vogl et al (2018) <https://doi.org/10.1016/j.jclepro.2018.08.279>`_
+elec_DRI,MWh/tSt,float,The electricity consumed in Direct Reduced Iron (DRI) shaft. From `HYBRIT brochure <https://ssabwebsitecdn.azureedge.net/-/media/hybrit/files/hybrit_brochure.pdf>`_
+Al_primary_fraction,--,Dictionary with planning horizons as keys.,The fraction of aluminium produced via the primary route versus scrap. Current fraction is 0.4
 MWh_NH3_per_tNH3,LHV,float,The energy amount per ton of ammonia.
-MWh_CH4_per_tNH3_SMR,--,float,The energy amount of methane needed to produce a ton of ammonia using steam methane reforming (SMR). Value derived from 2012's demand from https://ec.europa.eu/docsroom/documents/4165/attachments/1/translations/en/renditions/pdf
+MWh_CH4_per_tNH3_SMR,--,float,The energy amount of methane needed to produce a ton of ammonia using steam methane reforming (SMR). Value derived from 2012's demand from `Center for European Policy Studies (2008) <https://ec.europa.eu/docsroom/documents/4165/attachments/1/translations/en/renditions/pdf>`_
 MWh_elec_per_tNH3_SMR,--,float,"The energy amount of electricity needed to produce a ton of ammonia using steam methane reforming (SMR). same source, assuming 94-6% split methane-elec of total energy demand 11.5 MWh/tNH3"
-MWh_H2_per_tNH3_electrolysis,--,float,"The energy amount of hydrogen needed to produce a ton of ammonia using Haber–Bosch process. From https://doi.org/10.1016/j.joule.2018.04.017, around 0.197 tH2/tHN3 (>3/17 since some H2 lost and used for energy)"
-MWh_elec_per_tNH3_electrolysis,--,float,The energy amount of electricity needed to produce a ton of ammonia using Haber–Bosch process. From https://doi.org/10.1016/j.joule.2018.04.017 Table 13 (air separation and HB)
-MWh_NH3_per_MWh_H2_cracker,--,float,The energy amount of amonia needed to produce an energy amount hydrogen using ammonia cracker. https://github.com/euronion/trace/blob/44a5ff8401762edbef80eff9cfe5a47c8d3c8be4/data/efficiencies.csv
-NH3_process_emissions,MtCO2/a,float,The emission of ammonia production from steam methane reforming (SMR)
-petrochemical_process_emissions,MtCO2/a,float,The emission of petrochemical production
+MWh_H2_per_tNH3_electrolysis,--,float,"The energy amount of hydrogen needed to produce a ton of ammonia using Haber–Bosch process. From  `Wang et al (2018) <https://doi.org/10.1016/j.joule.2018.04.017>`_, Base value assumed around 0.197 tH2/tHN3 (>3/17 since some H2 lost and used for energy)"
+MWh_elec_per_tNH3_electrolysis,--,float,"The energy amount of electricity needed to produce a ton of ammonia using Haber–Bosch process. From  `Wang et al (2018) <https://doi.org/10.1016/j.joule.2018.04.017>`_, Table 13 (air separation and HB)"
+MWh_NH3_per_MWh_H2_cracker,--,float,The energy amount of amonia needed to produce an energy amount hydrogen using ammonia cracker
+NH3_process_emissions,MtCO2/a,float,The emission of ammonia production from steam methane reforming (SMR). From UNFCCC for 2015 for EU28
+petrochemical_process_emissions,MtCO2/a,float,The emission of petrochemical production. From UNFCCC for 2015 for EU28
 HVC_primary_fraction,--,float,The fraction of today's high value chemicals (HVC) produced via primary route
 HVC_mechanical_recycling_fraction,--,float,The fraction of today's high value chemicals (HVC) produced using mechanical recycling
 HVC_chemical_recycling_fraction,--,float,The fraction of today's high value chemicals (HVC) produced using chemical recycling
-HVC_production_today,MtHVC/a,float,The amount of high value chemicals (HVC) produced
-MWh_elec_per_tHVC_mechanical_recycling,--,float,The energy amount of electricity needed to produce a ton of high value chemical (HVC) using mechanical recycling
-MWh_elec_per_tHVC_chemical_recycling,--,float,The energy amount of electricity needed to produce a ton of high value chemical (HVC) using chemical recycling
-chlorine_production_today,MtCl/a,float,The amount of chlorine produced
-MWh_elec_per_tCl,--,float,The energy amount of electricity needed to produce a ton of chlorine
-MWh_H2_per_tCl,--,float,The energy amount of hydrogen needed to produce a ton of chlorine
-methanol_production_today,MtMeOH/a,float,The amount of methanol produced
-MWh_elec_per_tMeOH,--,float,The energy amount of electricity needed to produce a ton of methanol
-MWh_CH4_per_tMeOH,--,float,The energy amount of methane needed to produce a ton of methanol
-hotmaps_locate_missing,--,true or false,Locate industrial sites without valid locations based on city and countries.
-reference_year,--,year,
+HVC_production_today,MtHVC/a,float,"The amount of high value chemicals (HVC) produced. This includes ethylene, propylene and BTX. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, Figure 16, page 107"
+MWh_elec_per_tHVC_mechanical_recycling,MWh/tHVC,float,"The energy amount of electricity needed to produce a ton of high value chemical (HVC) using mechanical recycling. From SI of `Meys et al (2020) <https://doi.org/10.1016/j.resconrec.2020.105010>`_, Table S5, for HDPE, PP, PS, PET. LDPE would be 0.756."
+MWh_elec_per_tHVC_chemical_recycling,MWh/tHVC,float,"The energy amount of electricity needed to produce a ton of high value chemical (HVC) using chemical recycling. Value are based on pyrolysis and electric steam cracking. From `Material Economics (2019) <https://materialeconomics.com/latest-updates/industrial-transformation-2050>`_, page 125"
+chlorine_production_today,MtCl/a,float,"The amount of chlorine produced. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, Table 7, page 43"
+MWh_elec_per_tCl,MWh/tCl,float,"The energy amount of electricity needed to produce a ton of chlorine. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, Table 6 page 43"
+MWh_H2_per_tCl,MWhH2/tCl,float,"The energy amount of hydrogen needed to produce a ton of chlorine. The value is negative since hydrogen produced in chloralkali process. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, page 43"
+methanol_production_today,MtMeOH/a,float,"The amount of methanol produced. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, page 62"
+MWh_elec_per_tMeOH,MWh/tMeOH,float,"The energy amount of electricity needed to produce a ton of methanol. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, Table 14, page 65"
+MWh_CH4_per_tMeOH,MWhCH4/tMeOH,float,"The energy amount of methane needed to produce a ton of methanol. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, Table 14, page 65"
+hotmaps_locate_missing,--,"{true,false}",Locate industrial sites without valid locations based on city and countries.
+reference_year,year,YYYY,The year used as the baseline for industrial energy demand and production. Data extracted from `JRC-IDEES 2015 <https://data.jrc.ec.europa.eu/dataset/jrc-10110-10001>`_
diff --git a/doc/configtables/sector.csv b/doc/configtables/sector.csv
index 7a6f3c14..cb6c8115 100644
--- a/doc/configtables/sector.csv
+++ b/doc/configtables/sector.csv
@@ -2,9 +2,9 @@
 district_heating,--,,`prepare_sector_network.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/prepare_sector_network.py>`_
 -- potential,--,float,maximum fraction of urban demand which can be supplied by district heating increase of today's district heating demand to potential maximum district heating share
 -- progress,--,Dictionary with planning horizons as keys.,Progress = 0 means today's district heating share. Progress = 1 means maximum fraction of urban demand is supplied by district heating
--- district_heating_loss,--,float,
+-- district_heating_loss,--,float,Percentage increase in district heat demand in urban central due to heat losses
 cluster_heat_buses,--,"{true, false}",cluster residential and service heat buses in `prepare_sector_network.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/prepare_sector_network.py>`_  to one to save memory.
-bev_dsm_restriction_value,--,float,Adding a stage of charge (SOC) limit  for battery electric vehicles (BEV) to manage its own energy demand (DSM). Located in `build_transport_demand.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/build_transport_demand.py>`_.
+bev_dsm_restriction_value,--,float,Adding a stage of charge (SOC) limit  for battery electric vehicles (BEV) to manage its own energy demand (DSM). Located in `build_transport_demand.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/build_transport_demand.py>`_. Set to 0 for no restriction on BEV DSM
 bev_dsm_restriction_time,--,float,Time at which SOC of BEV has to be dsm_restriction_value
 transport_heating_deadband_upper,°C,float,"The minimum temperature in the vehicle. At lower temperatures, the energy required for heating in the vehicle increases."
 transport_heating_deadband_lower,°C,float,"The maximum temperature in the vehicle. At higher temperatures, the energy required for cooling in the vehicle increases."
@@ -16,54 +16,54 @@ bev_dsm,--,"{true, false}",Add the option for battery electric vehicles (BEV) to
 bev_availability,--,float,The percentage for battery electric vehicles (BEV) that are able to do demand side management (DSM)
 bev_energy,--,float,The average size of battery electric vehicles (BEV) in MWh
 bev_charge_efficiency,--,float,Battery electric vehicles (BEV) charge and discharge efficiency
-bev_plug_to_wheel_efficiency,km/kWh,float,The distance battery electric vehicles (BEV) can travel in km per kWh of energy charge in battery.  Base value comes from Tesla Model S https://www.fueleconomy.gov/feg/
+bev_plug_to_wheel_efficiency,km/kWh,float,The distance battery electric vehicles (BEV) can travel in km per kWh of energy charge in battery.  Base value comes from `Tesla Model S <https://www.fueleconomy.gov/feg/>`_
 bev_charge_rate,MWh,float,The power consumption for one electric vehicle (EV) in MWh. Value derived from 3-phase charger with 11 kW.
 bev_avail_max,--,float,The maximum percentage plugged-in availability for passenger electric vehicles.
 bev_avail_mean,--,float,The average percentage plugged-in availability for passenger electric vehicles.
 v2g,--,"{true, false}",Allows feed-in to grid from EV battery
 land_transport_fuel_cell_share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses fuel cells in a given year
 land_transport_electric_share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses electric vehicles (EV) in a given year
-land_transport_ice_share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses internal combustion engines (ICE) in a given year
+land_transport_ice_share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses internal combustion engines (ICE) in a given year. What is not EV or FCEV is oil-fuelled ICE.
 transport_fuel_cell_efficiency,--,float,The H2 conversion efficiencies of fuel cells in transport
 transport_internal_combustion_efficiency,--,float,The oil conversion efficiencies of internal combustion engine (ICE) in transport
 agriculture_machinery_electric_share,--,float,The percentage for agricultural machinery that uses electricity
 agriculture_machinery_oil_share,--,float,The percentage for agricultural machinery that uses oil
 agriculture_machinery_fuel_efficiency,--,float,The efficiency of electric-powered machinery in the conversion of electricity to meet agricultural needs.
 agriculture_machinery_electric_efficiency,--,float,The efficiency of oil-powered machinery in the conversion of oil to meet agricultural needs.
-MWh_MeOH_per_MWh_H2,LHV,float,The energy amount of the produced methanol per energy amount of hydrogen. source: DECHEMA (2017): Low carbon energy and feedstock for the European chemical industry page 64.
-MWh_MeOH_per_tCO2,LHV,float,The energy amount of the produced methanol per ton of CO2
-MWh_MeOH_per_MWh_e,LHV,float,The energy amount of the produced methanol per energy amount of electricity
+MWh_MeOH_per_MWh_H2,LHV,float,"The energy amount of the produced methanol per energy amount of hydrogen. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, page 64."
+MWh_MeOH_per_tCO2,LHV,float,"The energy amount of the produced methanol per ton of CO2. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, page 64."
+MWh_MeOH_per_MWh_e,LHV,float,"The energy amount of the produced methanol per energy amount of electricity. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, page 64."
 shipping_hydrogen_liquefaction,--,"{true, false}",Consider whether to include liquefaction costs for shipping H2 demand.
 shipping_hydrogen_share,--,Dictionary with planning horizons as keys.,The share of ships powered by hydrogen in a given year
 shipping_methanol_share,--,Dictionary with planning horizons as keys.,The share of ships powered by methanol in a given year
 shipping_oil_share,--,Dictionary with planning horizons as keys.,The share of ships powered by oil in a given year
-shipping_methanol_efficiency,--,float,The efficiency of methanol-powered ships in the conversion of methanol to meet shipping needs.
-shipping_oil_efficiency,--,float,The efficiency of oil-powered ships in the conversion of oil to meet shipping needs.
+shipping_methanol_efficiency,--,float,"The efficiency of methanol-powered ships in the conversion of methanol to meet shipping needs (propulsion). The efficiency increase from oil can be 10-15% higher according to the `IEA <https://www.iea-amf.org/app/webroot/files/file/Annex%20Reports/AMF_Annex_56.pdf>`_,"
+shipping_oil_efficiency,--,float,The efficiency of oil-powered ships in the conversion of oil to meet shipping needs (propulsion). Base value derived from 2011
 aviation_demand_factor,--,float,The proportion of demand for aviation compared to today's
 HVC_demand_factor,--,float,The proportion of demand for high-value chemicals compared to today's
-time_dep_hp_cop,--,"{true, false}",
-heat_pump_sink_T,°C,float,
-reduce_space_heat_exogenously,--,"{true, false}",
-reduce_space_heat_exogenously_factor,--,Dictionary with planning horizons as keys.,
+time_dep_hp_cop,--,"{true, false}",Consider the time dependent coefficient of performance (COP) of the heat pump
+heat_pump_sink_T,°C,float,The temperature heat sink used in heat pumps based on DTU / large area radiators. The value is conservatively high to cover hot water and space heating in poorly-insulated buildings
+reduce_space_heat_exogenously,--,"{true, false}",Influence on space heating demand by a certain factor (applied before losses in district heating). 
+reduce_space_heat_exogenously_factor,--,Dictionary with planning horizons as keys.,"A positive factor can mean renovation or demolition of a building. If the factor is negative, it can mean an increase in floor area, increased thermal comfort, population growth. The default factors are determined by the `Eurocalc Homes and buildings decarbonization scenario <http://tool.european-calculator.eu/app/buildings/building-types-area/?levers=1ddd4444421213bdbbbddd44444ffffff11f411111221111211l212221>`_"
 retrofitting,,,
 -- retro_endogen,--,"{true, false}",Add retrofitting as an endogenous system which co-optimise space heat savings.
--- cost_factor,,,Weight costs for building renovation
--- interest_rate,,,The interest rate for investment in building components
+-- cost_factor,--,float,Weight costs for building renovation
+-- interest_rate,--,float,The interest rate for investment in building components
 -- annualise_cost,--,"{true, false}",Annualise the investment costs of retrofitting
 -- tax_weighting,--,"{true, false}",Weight the costs of retrofitting depending on taxes in countries
 -- construction_index,--,"{true, false}",Weight the costs of retrofitting depending on labour/material costs per country
 tes,--,"{true, false}",Add option for storing thermal energy in large water pits associated with district heating systems and individual thermal energy storage (TES)
-tes_tau,,,
--- decentral,,,
--- central,,,
+tes_tau,,,The time constant used to calculate the decay of thermal energy in thermal energy storage (TES): 1- :math:`e^{-1/24τ}`.
+-- decentral,days,float,The time constant in decentralized thermal energy storage (TES)
+-- central,days,float,The time constant in centralized thermal energy storage (TES)
 boilers,--,"{true, false}",Add option for transforming electricity into heat using resistive heater
 oil_boilers,--,"{true, false}",Add option for transforming oil into heat using boilers
 biomass_boiler,--,"{true, false}",Add option for transforming biomass into heat using boilers
 chp,--,"{true, false}",Add option for using Combined Heat and Power (CHP)
 micro_chp,--,"{true, false}",Add option for using Combined Heat and Power (CHP) for decentral areas.
-solar_thermal,--,"{true, false}",Add option for using solar to generate heat.
-solar_cf_correction,,,
-marginal_cost_storage,,,
+solar_thermal,--,"{true, false}",Add option for using solar thermal to generate heat.
+solar_cf_correction,--,float,The correction factor for the value provided by the solar thermal profile calculations
+marginal_cost_storage,currency/MWh ,float,The marginal cost of discharging batteries in distributed grids
 methanation,--,"{true, false}",Add option for transforming hydrogen and CO2 into methane using methanation.
 helmeth,--,"{true, false}",Add option for transforming power into gas using HELMETH (Integrated High-Temperature ELectrolysis and METHanation for Effective Power to Gas Conversion)
 coal_cc,--,"{true, false}",Add option for coal CHPs with carbon capture
@@ -75,21 +75,21 @@ hydrogen_turbine,--,"{true, false}",Add option to include hydrogen turbine for r
 SMR,--,"{true, false}",Add option for transforming natural gas into hydrogen and CO2 using Steam Methane Reforming (SMR)
 regional_co2_sequestration_potential,,,
 -- enable,--,"{true, false}",Add option for regionally-resolved geological carbon dioxide sequestration potentials based on `CO2StoP <https://setis.ec.europa.eu/european-co2-storage-database_en>`_.
--- attribute,,,
--- include_onshore,,"{true, false}",Add options for including onshore sequestration potentials
--- min_size,,float,Any sites with lower potential than this value will be excluded
--- max_size,,float,The maximum sequestration potential for any one site.
--- years_of_storage,,float,The years until potential exhausted at optimised annual rate
+-- attribute,--,string,Name of the attribute for the sequestration potential
+-- include_onshore,--,"{true, false}",Add options for including onshore sequestration potentials
+-- min_size,Gt ,float,Any sites with lower potential than this value will be excluded
+-- max_size,Gt ,float,The maximum sequestration potential for any one site.
+-- years_of_storage,years,float,The years until potential exhausted at optimised annual rate
 co2_sequestration_potential,MtCO2/a,float,The potential of sequestering CO2 in Europe per year
-co2_sequestration_cost,EUR/tCO2,float,The cost of sequestering a ton of CO2
+co2_sequestration_cost,currency/tCO2,float,The cost of sequestering a ton of CO2
 co2_spatial,--,"{true, false}","„Add option to spatially resolve carrier representing stored carbon dioxide. This allows for more detailed modelling of CCUTS, e.g. regarding the capturing of industrial process emissions, usage as feedstock for electrofuels, transport of carbon dioxide, and geological sequestration sites.“"
 co2network,--,"{true, false}",Add option for planning a new carbon dioxide network
-cc_fraction,,,The default fraction of CO2 captured with post-combustion capture
+cc_fraction,--,float,The default fraction of CO2 captured with post-combustion capture
 hydrogen_underground_storage,--,"{true, false}",Add options for storing hydrogen underground. Storage potential depends regionally.
 hydrogen_underground_storage_locations,,"{onshore, nearshore, offshore}","„The location where hydrogen underground storage can be located. Onshore, nearshore, offshore means it must be located more than 50 km away from the sea, within 50 km of the sea, or within the sea itself respectively.“"
-ammonia,--,"{true, false, regional}","Add ammonia as a carrrier. It can be either true (copperplated NH3) or ""regional"" (regionalised NH3 without network)"
-min_part_load_fischer_tropsch,,,
-min_part_load_methanolisation,,,
+ammonia,--,"{true, false, regional}","Add ammonia as a carrrier. It can be either true (copperplated NH3), false (no NH3 carrier) or ""regional"" (regionalised NH3 without network)"
+min_part_load_fischer_tropsch,per unit of p_nom ,float,The minimum unit dispatch (p_min_pu) for the Fischer-Tropsch process
+min_part_load_methanolisation,per unit of p_nom ,float,The minimum unit dispatch (p_min_pu) for the methanolisation process
 use_fischer_tropsch_waste_heat,--,"{true, false}",Add option for using waste heat of Fischer Tropsch in district heating networks
 use_fuel_cell_waste_heat,--,"{true, false}",Add option for using waste heat of fuel cells in district heating networks
 use_electrolysis_waste_heat,--,"{true, false}",Add option for using waste heat of electrolysis in district heating networks
@@ -97,9 +97,10 @@ electricity_distribution_grid,--,"{true, false}",Add a electricity distribution
 electricity_distribution_grid_cost_factor,,,Multiplies the investment cost of the electricity distribution grid in data/costs.csv
 electricity_grid_connection,--,"{true, false}",Add the cost of electricity grid connection for onshore wind and solar
 H2_network,--,"{true, false}",Add option for new hydrogen pipelines
-gas_network,--,"{true, false}","Add natural gas infrastructure, incl. LNG terminals, production and entry-points"
-H2_retrofit,--,"{true, false}",Add option for retrofiting existing pipelines to transport hydrogen
-H2_retrofit_capacity_per_CH4,,,
+gas_network,--,"{true, false}","Add existing natural gas infrastructure, incl. LNG terminals, production and entry-points. The existing gas network is added with a lossless transport model. A length-weighted `k-edge augmentation algorithm   <https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation>`_   can be run to add new candidate gas pipelines such that all regions of the model can be connected to the gas network. When activated, all the gas demands are regionally disaggregated as well."
+H2_retrofit,--,"{true, false}","Add option for retrofiting existing pipelines to transport hydrogen. The reasoning is in accordance with the `hydrogen backbone strategy (April, 2020) p.15 <https://gasforclimate2050.eu/wp-content/uploads/2020/07/2020_European-Hydrogen-Backbone_Report.pdf>`_. 60% of original natural gas capacity could be used in cost-optimal case as H2 capacity."
+H2_retrofit_capacity_per_CH4,--,float,The ratio for H2 capacity per original CH4 capacity of retrofitted pipelines 
+gas_network_connectivity_upgrade ,--,float,The number of desired edge connectivity (k) in the length-weighted `k-edge augmentation algorithm <https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation>`_ used for the gas network
 gas_distribution_grid,--,"{true, false}",Add a gas distribution grid
 gas_distribution_grid_cost_factor,,,Multiplies the investment cost of the gas distribution grid in data/costs.csv
 biomass_spatial,--,"{true, false}",Add option for resolving biomass demand regionally
diff --git a/doc/configuration.rst b/doc/configuration.rst
index cd926b6a..1aa6e76d 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -191,6 +191,9 @@ Switches for some rules and optional features.
    :widths: 25,7,22,30
    :file: configtables/electricity.csv
 
+.. note::
+   Wind is the Fueltype in powerplantmatching, onwind, offwind-{ac,dc} the carrier in PyPSA-Eur
+
 .. _atlite_cf:
 
 ``atlite``
@@ -470,7 +473,7 @@ The list of available biomass is given by the category in `ENSPRESO_BIOMASS <htt
 =======================
 
 .. note::
-   Only used for sector-coupling studies. The value for grouping years are only used in myopic scenarios.
+   Only used for sector-coupling studies. The value for grouping years are only used in myopic or perfect scenarios.
 
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
@@ -508,9 +511,6 @@ The list of available biomass is given by the category in `ENSPRESO_BIOMASS <htt
 .. note::
    Only used for sector-coupling studies.
 
-.. warning::
-   More comprehensive documentation for this segment will be released soon.
-
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml
    :start-at: industry:

From 0664f1050f7a7d79740f6ceb4490719d3d98d017 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 4 Jul 2023 23:58:05 +0000
Subject: [PATCH 152/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 config/config.default.yaml          | 10 +++++-----
 doc/configtables/.~lock.sector.csv# |  2 +-
 doc/configtables/sector.csv         |  4 ++--
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/config/config.default.yaml b/config/config.default.yaml
index f3b69e4a..6e5fba77 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -323,7 +323,7 @@ existing_capacities:
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#sector
 sector:
   district_heating:
-    potential: 0.6 
+    potential: 0.6
     progress:
       2020: 0.0
       2030: 0.3
@@ -434,7 +434,7 @@ sector:
   regional_co2_sequestration_potential:
     enable: false
     attribute: 'conservative estimate Mt'
-    include_onshore: false 
+    include_onshore: false
     min_size: 3
     max_size: 25
     years_of_storage: 25
@@ -455,12 +455,12 @@ sector:
   use_fuel_cell_waste_heat: true
   use_electrolysis_waste_heat: false
   electricity_distribution_grid: true
-  electricity_distribution_grid_cost_factor: 1.0 
+  electricity_distribution_grid_cost_factor: 1.0
   electricity_grid_connection: true
   H2_network: true
   gas_network: false
   H2_retrofit: false
-  H2_retrofit_capacity_per_CH4: 0.6 
+  H2_retrofit_capacity_per_CH4: 0.6
   gas_network_connectivity_upgrade: 1
   gas_distribution_grid: true
   gas_distribution_grid_cost_factor: 1.0
@@ -489,7 +489,7 @@ industry:
     2040: 0.4
     2045: 0.7
     2050: 1
-  H2_DRI: 1.7 
+  H2_DRI: 1.7
   elec_DRI: 0.322
   Al_primary_fraction:
     2020: 0.4
diff --git a/doc/configtables/.~lock.sector.csv# b/doc/configtables/.~lock.sector.csv#
index b614fb4e..24f46f24 100644
--- a/doc/configtables/.~lock.sector.csv#
+++ b/doc/configtables/.~lock.sector.csv#
@@ -1 +1 @@
-,DESKTOP-2RNCH2B/kunde,,05.07.2023 01:09,file:///C:/Users/kunde/AppData/Roaming/LibreOffice/4;
\ No newline at end of file
+,DESKTOP-2RNCH2B/kunde,,05.07.2023 01:09,file:///C:/Users/kunde/AppData/Roaming/LibreOffice/4;
diff --git a/doc/configtables/sector.csv b/doc/configtables/sector.csv
index cb6c8115..29805273 100644
--- a/doc/configtables/sector.csv
+++ b/doc/configtables/sector.csv
@@ -43,7 +43,7 @@ aviation_demand_factor,--,float,The proportion of demand for aviation compared t
 HVC_demand_factor,--,float,The proportion of demand for high-value chemicals compared to today's
 time_dep_hp_cop,--,"{true, false}",Consider the time dependent coefficient of performance (COP) of the heat pump
 heat_pump_sink_T,°C,float,The temperature heat sink used in heat pumps based on DTU / large area radiators. The value is conservatively high to cover hot water and space heating in poorly-insulated buildings
-reduce_space_heat_exogenously,--,"{true, false}",Influence on space heating demand by a certain factor (applied before losses in district heating). 
+reduce_space_heat_exogenously,--,"{true, false}",Influence on space heating demand by a certain factor (applied before losses in district heating).
 reduce_space_heat_exogenously_factor,--,Dictionary with planning horizons as keys.,"A positive factor can mean renovation or demolition of a building. If the factor is negative, it can mean an increase in floor area, increased thermal comfort, population growth. The default factors are determined by the `Eurocalc Homes and buildings decarbonization scenario <http://tool.european-calculator.eu/app/buildings/building-types-area/?levers=1ddd4444421213bdbbbddd44444ffffff11f411111221111211l212221>`_"
 retrofitting,,,
 -- retro_endogen,--,"{true, false}",Add retrofitting as an endogenous system which co-optimise space heat savings.
@@ -99,7 +99,7 @@ electricity_grid_connection,--,"{true, false}",Add the cost of electricity grid
 H2_network,--,"{true, false}",Add option for new hydrogen pipelines
 gas_network,--,"{true, false}","Add existing natural gas infrastructure, incl. LNG terminals, production and entry-points. The existing gas network is added with a lossless transport model. A length-weighted `k-edge augmentation algorithm   <https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation>`_   can be run to add new candidate gas pipelines such that all regions of the model can be connected to the gas network. When activated, all the gas demands are regionally disaggregated as well."
 H2_retrofit,--,"{true, false}","Add option for retrofiting existing pipelines to transport hydrogen. The reasoning is in accordance with the `hydrogen backbone strategy (April, 2020) p.15 <https://gasforclimate2050.eu/wp-content/uploads/2020/07/2020_European-Hydrogen-Backbone_Report.pdf>`_. 60% of original natural gas capacity could be used in cost-optimal case as H2 capacity."
-H2_retrofit_capacity_per_CH4,--,float,The ratio for H2 capacity per original CH4 capacity of retrofitted pipelines 
+H2_retrofit_capacity_per_CH4,--,float,The ratio for H2 capacity per original CH4 capacity of retrofitted pipelines
 gas_network_connectivity_upgrade ,--,float,The number of desired edge connectivity (k) in the length-weighted `k-edge augmentation algorithm <https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation>`_ used for the gas network
 gas_distribution_grid,--,"{true, false}",Add a gas distribution grid
 gas_distribution_grid_cost_factor,,,Multiplies the investment cost of the gas distribution grid in data/costs.csv

From 9c6e5e0ed055b4ab0196ce26552df6ae9920a3c2 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Wed, 5 Jul 2023 11:34:07 +0200
Subject: [PATCH 153/176] config: remove unneeded plotting entries

---
 config/config.default.yaml    | 42 -----------------------------------
 doc/configtables/plotting.csv |  9 --------
 scripts/solve_network.py      |  9 ++++----
 3 files changed, 5 insertions(+), 55 deletions(-)

diff --git a/config/config.default.yaml b/config/config.default.yaml
index 6e5fba77..57477618 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -665,48 +665,6 @@ plotting:
   energy_max: 20000
   energy_min: -20000
   energy_threshold: 50.
-  vre_techs:
-  - onwind
-  - offwind-ac
-  - offwind-dc
-  - solar
-  - ror
-  renewable_storage_techs:
-  - PHS
-  - hydro
-  conv_techs:
-  - OCGT
-  - CCGT
-  - Nuclear
-  - Coal
-  storage_techs:
-  - hydro+PHS
-  - battery
-  - H2
-  load_carriers:
-  - AC load
-  AC_carriers:
-  - AC line
-  - AC transformer
-  link_carriers:
-  - DC line
-  - Converter AC-DC
-  heat_links:
-  - heat pump
-  - resistive heater
-  - CHP heat
-  - CHP electric
-  - gas boiler
-  - central heat pump
-  - central resistive heater
-  - central CHP heat
-  - central CHP electric
-  - central gas boiler
-  heat_generators:
-  - gas boiler
-  - central gas boiler
-  - solar thermal collector
-  - central solar thermal collector
 
   nice_names:
     OCGT: "Open-Cycle Gas"
diff --git a/doc/configtables/plotting.csv b/doc/configtables/plotting.csv
index 82bf7af6..ed5d9c9f 100644
--- a/doc/configtables/plotting.csv
+++ b/doc/configtables/plotting.csv
@@ -7,13 +7,4 @@ energy_max,TWh,float,Upper y-axis limit in energy bar plots.
 energy_min,TWh,float,Lower y-axis limit in energy bar plots.
 energy_threshold,TWh,float,Threshold below which technologies will not be shown in energy bar plots.
 tech_colors,--,carrier -> HEX colour code,Mapping from network ``carrier`` to a colour (`HEX colour code <https://en.wikipedia.org/wiki/Web_colors#Hex_triplet>`_).
-vre_techs ,--,subset of carriers,Value not in use
-renewable_storage_techs ,--,subset of carriers,Value not in use
-conv_techs ,--,subset of carriers,Value in use with unworthy significance. Used as the definition of conventional carrier in solve_network
-storage_techs ,--,subset of carriers,Value not in use
-load_carriers ,--,subset of carriers,Value not in use
-AC_carriers ,--,subset of carriers,Value not in use
-link_carriers ,--,subset of carriers,Value not in use
-heat_links ,--,subset of carriers,Value not in use
-heat_generators ,--,subset of carriers,Value not in use
 nice_names,--,str -> str,Mapping from network ``carrier`` to a more readable name.
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index 58f4db72..37dea162 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -378,13 +378,14 @@ def add_SAFE_constraints(n, config):
     peakdemand = n.loads_t.p_set.sum(axis=1).max()
     margin = 1.0 + config["electricity"]["SAFE_reservemargin"]
     reserve_margin = peakdemand * margin
-    # TODO: do not take this from the plotting config!
-    conv_techs = config["plotting"]["conv_techs"]
-    ext_gens_i = n.generators.query("carrier in @conv_techs & p_nom_extendable").index
+    conventional_carriers = config["electricity"]["conventional_carriers"]
+    ext_gens_i = n.generators.query(
+        "carrier in @conventional_carriers & p_nom_extendable"
+    ).index
     p_nom = n.model["Generator-p_nom"].loc[ext_gens_i]
     lhs = p_nom.sum()
     exist_conv_caps = n.generators.query(
-        "~p_nom_extendable & carrier in @conv_techs"
+        "~p_nom_extendable & carrier in @conventional_carriers"
     ).p_nom.sum()
     rhs = reserve_margin - exist_conv_caps
     n.model.add_constraints(lhs >= rhs, name="safe_mintotalcap")

From 3ea8b26ae01fb4dc35cde2f66dea8ffd069c344f Mon Sep 17 00:00:00 2001
From: Fabian Hofmann <fab.hof@gmx.de>
Date: Thu, 6 Jul 2023 15:01:00 +0200
Subject: [PATCH 154/176] fix carrier assignment

---
 scripts/add_electricity.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 2f6251f0..7cea165b 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -161,7 +161,7 @@ def sanitize_carriers(n, config):
 
     for c in n.iterate_components():
         if "carrier" in c.df:
-            add_missing_carriers(n, c.df)
+            add_missing_carriers(n, c.df.carrier)
 
     carrier_i = n.carriers.index
     nice_names = (

From 26156943cf427418408f921e3078c3282c7c060f Mon Sep 17 00:00:00 2001
From: lisazeyen <35347358+lisazeyen@users.noreply.github.com>
Date: Mon, 10 Jul 2023 16:15:27 +0200
Subject: [PATCH 155/176] remove TODO comment

the clipping of the p_min_pu value cannot be removed since it is used for the retrofitting generators which have to follow the heat demand profile. Otherwise the network becomes infeasibe.
---
 scripts/solve_network.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index 58f4db72..1a7e4641 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -152,7 +152,7 @@ def prepare_network(
     if "clip_p_max_pu" in solve_opts:
         for df in (
             n.generators_t.p_max_pu,
-            n.generators_t.p_min_pu,  # TODO: check if this can be removed
+            n.generators_t.p_min_pu,  
             n.storage_units_t.inflow,
         ):
             df.where(df > solve_opts["clip_p_max_pu"], other=0.0, inplace=True)

From 5026b560700271bfba3de65172727205376e0307 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 10 Jul 2023 23:01:49 +0000
Subject: [PATCH 156/176] [pre-commit.ci] pre-commit autoupdate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

updates:
- [github.com/PyCQA/docformatter: v1.7.3 → v1.7.4](https://github.com/PyCQA/docformatter/compare/v1.7.3...v1.7.4)
---
 .pre-commit-config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 7bb35b53..cf811240 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -39,7 +39,7 @@ repos:
 
   # Make docstrings PEP 257 compliant
 - repo: https://github.com/PyCQA/docformatter
-  rev: v1.7.3
+  rev: v1.7.4
   hooks:
   - id: docformatter
     args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"]

From 6492b1a195e3e8868bce18b3ac13f4f0602820b3 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 10 Jul 2023 23:02:55 +0000
Subject: [PATCH 157/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/solve_network.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index 1a7e4641..f7ab724c 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -152,7 +152,7 @@ def prepare_network(
     if "clip_p_max_pu" in solve_opts:
         for df in (
             n.generators_t.p_max_pu,
-            n.generators_t.p_min_pu,  
+            n.generators_t.p_min_pu,
             n.storage_units_t.inflow,
         ):
             df.where(df > solve_opts["clip_p_max_pu"], other=0.0, inplace=True)

From d73452903190e9c422f85229656939470e37008b Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Wed, 12 Jul 2023 12:42:31 +0200
Subject: [PATCH 158/176] Apply Fabian Neumann suggestions from code review

Co-authored-by: Fabian Neumann <fabian.neumann@outlook.de>
---
 doc/configtables/electricity.csv         | 12 ++++---
 doc/configtables/energy.csv              |  9 +++--
 doc/configtables/existing_capacities.csv |  5 +--
 doc/configtables/industry.csv            | 13 ++++---
 doc/configtables/sector.csv              | 46 ++++++++++++++----------
 doc/configtables/solar-thermal.csv       |  3 +-
 6 files changed, 55 insertions(+), 33 deletions(-)

diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv
index 02dcbdde..828c34df 100644
--- a/doc/configtables/electricity.csv
+++ b/doc/configtables/electricity.csv
@@ -4,7 +4,8 @@ gaslimit,MWhth,float or false,Global gas usage limit
 co2limit,:math:`t_{CO_2-eq}/a`,float,Cap on total annual system carbon dioxide emissions
 co2base,:math:`t_{CO_2-eq}/a`,float,Reference value of total annual system carbon dioxide emissions if relative emission reduction target is specified in ``{opts}`` wildcard.
 agg_p_nom_limits,file,path,Reference to ``.csv`` file specifying per carrier generator nominal capacity constraints for individual countries if ``'CCL'`` is in ``{opts}`` wildcard. Defaults to ``data/agg_p_nom_minmax.csv``.
-operational_reserve,,,Settings for reserve requirements following like `GenX <https://genxproject.github.io/GenX/dev/core/#Reserves>`_
+operational_reserve,,,Settings for reserve requirements following `GenX <https://genxproject.github.io/GenX/dev/core/#Reserves>`_
+
 -- activate,bool,true or false,Whether to take operational reserve requirements into account during optimisation
 -- epsilon_load,--,float,share of total load
 -- epsilon_vres,--,float,share of total renewable supply
@@ -17,9 +18,12 @@ extendable_carriers,,,
 -- StorageUnit,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity.
 -- Store,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity.
 -- Link,--,Any subset of {'H2 pipeline'},Adds extendable links (H2 pipelines only) at every connection where there are lines or HVDC links without capacity limits and with zero initial capacity. Hydrogen pipelines require hydrogen storage to be modelled as ``Store``.
-powerplants_filter,--,"use `pandas.query <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.query.html>`_ strings here, e.g. Country not in ['Germany']",Filter query for the default powerplant database.
-custom_powerplants,--,"use `pandas.query <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.query.html>`_ strings here, e.g. Country in ['Germany']",Filter query for the custom powerplant database.
-conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to include in the model from ``resources/powerplants.csv``. If an included carrier is also listed in `extendable_carriers`, the capacity is taken as a lower bound."
+powerplants_filter,--,"use `pandas.query <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.query.html>`_ strings here, e.g. ``Country not in ['Germany']``",Filter query for the default powerplant database.
+
+custom_powerplants,--,"use `pandas.query <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.query.html>`_ strings here, e.g. ``Country in ['Germany']``",Filter query for the custom powerplant database.
+
+conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to include in the model from ``resources/powerplants.csv``. If an included carrier is also listed in ``extendable_carriers``, the capacity is taken as a lower bound."
+
 renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, hydro}",List of renewable generators to include in the model.
 estimate_renewable_capacities,,,
 -- enable,,bool,Activate routine to estimate renewable capacities
diff --git a/doc/configtables/energy.csv b/doc/configtables/energy.csv
index 640a9bc3..ff5e48cd 100644
--- a/doc/configtables/energy.csv
+++ b/doc/configtables/energy.csv
@@ -1,5 +1,8 @@
 ,Unit,Values,Description
 energy_totals_year ,--,"{1990,1995,2000,2005,2010,2011,…} ",The year for the sector energy use. The year must be avaliable in the Eurostat report
-base_emissions_year ,--,YYYY; e.g. ‚1990‘ ,'The base year for the sector emissions. See `European Environment Agency (EEA) <https://www.eea.europa.eu/data-and-maps/data/national-emissions-reported-to-the-unfccc-and-to-the-eu-greenhouse-gas-monitoring-mechanism-16>`_ downloaded 201228 (modified by EEA last on 201221)‘
-eurostat_report_year ,--,"{2016,2017,2018}","'The year in which the Eurostat report year are extracted. 2016 includes Bosnia and Herzegovina, 2017 does not‘"
-emissions ,--,"{CO2, All greenhouse gases - (CO2 equivalent)}",Specify which sectoral emissions are taken into account. Data derived from EEA.
+base_emissions_year ,--,"YYYY; e.g. 1990","The base year for the sector emissions. See `European Environment Agency (EEA) <https://www.eea.europa.eu/data-and-maps/data/national-emissions-reported-to-the-unfccc-and-to-the-eu-greenhouse-gas-monitoring-mechanism-16>`_."
+
+eurostat_report_year ,--,"{2016,2017,2018}","The publication year of the Eurostat report. 2016 includes Bosnia and Herzegovina, 2017 does not"
+
+emissions ,--,"{CO2, All greenhouse gases - (CO2 equivalent)}","Specify which sectoral emissions are taken into account. Data derived from EEA. Currently only CO2 is implemented."
+
diff --git a/doc/configtables/existing_capacities.csv b/doc/configtables/existing_capacities.csv
index 6b53d5f1..87519193 100644
--- a/doc/configtables/existing_capacities.csv
+++ b/doc/configtables/existing_capacities.csv
@@ -1,5 +1,6 @@
 ,Unit,Values,Description
-grouping_years_power ,--,An array of years,Intervals to group existing capacities for power
-grouping_years_heat ,--,An array of years below 2020,Intervals to group existing capacities for heat
+grouping_years_power ,--,A list of years,Intervals to group existing capacities for power
+grouping_years_heat ,--,A list of years below 2020,Intervals to group existing capacities for heat
+
 threshold_capacity ,MW,float,Capacities generators and links of below threshold are removed during add_existing_capacities
 conventional_carriers ,--,"Any subset of {uranium, coal, lignite, oil} ",List of conventional power plants to include in the sectoral network
diff --git a/doc/configtables/industry.csv b/doc/configtables/industry.csv
index 248ed4b5..358da5c8 100644
--- a/doc/configtables/industry.csv
+++ b/doc/configtables/industry.csv
@@ -1,6 +1,7 @@
 ,Unit,Values,Description
 St_primary_fraction,--,Dictionary with planning horizons as keys.,The fraction of steel produced via primary route versus secondary route (scrap+EAF). Current fraction is 0.6
-DRI_fraction,--,Dictionary with planning horizons as keys.,The fraction of the primary route converted to DRI + EAF
+DRI_fraction,--,Dictionary with planning horizons as keys.,The fraction of the primary route DRI + EAF
+
 H2_DRI,--,float,The hydrogen consumption in Direct Reduced Iron (DRI)  Mwh_H2 LHV/ton_Steel from 51kgH2/tSt in `Vogl et al (2018) <https://doi.org/10.1016/j.jclepro.2018.08.279>`_
 elec_DRI,MWh/tSt,float,The electricity consumed in Direct Reduced Iron (DRI) shaft. From `HYBRIT brochure <https://ssabwebsitecdn.azureedge.net/-/media/hybrit/files/hybrit_brochure.pdf>`_
 Al_primary_fraction,--,Dictionary with planning horizons as keys.,The fraction of aluminium produced via the primary route versus scrap. Current fraction is 0.4
@@ -12,12 +13,14 @@ MWh_elec_per_tNH3_electrolysis,--,float,"The energy amount of electricity needed
 MWh_NH3_per_MWh_H2_cracker,--,float,The energy amount of amonia needed to produce an energy amount hydrogen using ammonia cracker
 NH3_process_emissions,MtCO2/a,float,The emission of ammonia production from steam methane reforming (SMR). From UNFCCC for 2015 for EU28
 petrochemical_process_emissions,MtCO2/a,float,The emission of petrochemical production. From UNFCCC for 2015 for EU28
-HVC_primary_fraction,--,float,The fraction of today's high value chemicals (HVC) produced via primary route
-HVC_mechanical_recycling_fraction,--,float,The fraction of today's high value chemicals (HVC) produced using mechanical recycling
-HVC_chemical_recycling_fraction,--,float,The fraction of today's high value chemicals (HVC) produced using chemical recycling
+HVC_primary_fraction,--,float,The fraction of high value chemicals (HVC) produced via primary route
+HVC_mechanical_recycling_fraction,--,float,The fraction of high value chemicals (HVC) produced using mechanical recycling
+HVC_chemical_recycling_fraction,--,float,The fraction of high value chemicals (HVC) produced using chemical recycling
+
 HVC_production_today,MtHVC/a,float,"The amount of high value chemicals (HVC) produced. This includes ethylene, propylene and BTX. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, Figure 16, page 107"
 MWh_elec_per_tHVC_mechanical_recycling,MWh/tHVC,float,"The energy amount of electricity needed to produce a ton of high value chemical (HVC) using mechanical recycling. From SI of `Meys et al (2020) <https://doi.org/10.1016/j.resconrec.2020.105010>`_, Table S5, for HDPE, PP, PS, PET. LDPE would be 0.756."
-MWh_elec_per_tHVC_chemical_recycling,MWh/tHVC,float,"The energy amount of electricity needed to produce a ton of high value chemical (HVC) using chemical recycling. Value are based on pyrolysis and electric steam cracking. From `Material Economics (2019) <https://materialeconomics.com/latest-updates/industrial-transformation-2050>`_, page 125"
+MWh_elec_per_tHVC_chemical_recycling,MWh/tHVC,float,"The energy amount of electricity needed to produce a ton of high value chemical (HVC) using chemical recycling. The default value is based on pyrolysis and electric steam cracking. From `Material Economics (2019) <https://materialeconomics.com/latest-updates/industrial-transformation-2050>`_, page 125"
+
 chlorine_production_today,MtCl/a,float,"The amount of chlorine produced. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, Table 7, page 43"
 MWh_elec_per_tCl,MWh/tCl,float,"The energy amount of electricity needed to produce a ton of chlorine. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, Table 6 page 43"
 MWh_H2_per_tCl,MWhH2/tCl,float,"The energy amount of hydrogen needed to produce a ton of chlorine. The value is negative since hydrogen produced in chloralkali process. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, page 43"
diff --git a/doc/configtables/sector.csv b/doc/configtables/sector.csv
index 29805273..7f7efed4 100644
--- a/doc/configtables/sector.csv
+++ b/doc/configtables/sector.csv
@@ -1,10 +1,11 @@
 ,Unit,Values,Description
 district_heating,--,,`prepare_sector_network.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/prepare_sector_network.py>`_
--- potential,--,float,maximum fraction of urban demand which can be supplied by district heating increase of today's district heating demand to potential maximum district heating share
--- progress,--,Dictionary with planning horizons as keys.,Progress = 0 means today's district heating share. Progress = 1 means maximum fraction of urban demand is supplied by district heating
+-- potential,--,float,maximum fraction of urban demand which can be supplied by district heating
+-- progress,--,Dictionary with planning horizons as keys., Increase of today's district heating demand to potential maximum district heating share. Progress = 0 means today's district heating share. Progress = 1 means maximum fraction of urban demand is supplied by district heating
 -- district_heating_loss,--,float,Percentage increase in district heat demand in urban central due to heat losses
-cluster_heat_buses,--,"{true, false}",cluster residential and service heat buses in `prepare_sector_network.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/prepare_sector_network.py>`_  to one to save memory.
-bev_dsm_restriction_value,--,float,Adding a stage of charge (SOC) limit  for battery electric vehicles (BEV) to manage its own energy demand (DSM). Located in `build_transport_demand.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/build_transport_demand.py>`_. Set to 0 for no restriction on BEV DSM
+cluster_heat_buses,--,"{true, false}",Cluster residential and service heat buses in `prepare_sector_network.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/prepare_sector_network.py>`_  to one to save memory.
+
+bev_dsm_restriction_value,--,float,Adds a lower state of charge (SOC) limit for battery electric vehicles (BEV) to manage its own energy demand (DSM). Located in `build_transport_demand.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/build_transport_demand.py>`_. Set to 0 for no restriction on BEV DSM
 bev_dsm_restriction_time,--,float,Time at which SOC of BEV has to be dsm_restriction_value
 transport_heating_deadband_upper,°C,float,"The minimum temperature in the vehicle. At lower temperatures, the energy required for heating in the vehicle increases."
 transport_heating_deadband_lower,°C,float,"The maximum temperature in the vehicle. At higher temperatures, the energy required for cooling in the vehicle increases."
@@ -12,7 +13,8 @@ ICE_lower_degree_factor,--,float,Percentage increase in energy demand in interna
 ICE_upper_degree_factor,--,float,Percentage increase in energy demand in internal combustion engine (ICE) for each degree difference between the hot environment and the maximum temperature.
 EV_lower_degree_factor,--,float,Percentage increase in energy demand in electric vehicles (EV) for each degree difference between the cold environment and the minimum temperature.
 EV_upper_degree_factor,--,float,Percentage increase in energy demand in electric vehicles (EV) for each degree difference between the hot environment and the maximum temperature.
-bev_dsm,--,"{true, false}",Add the option for battery electric vehicles (BEV) to manage its own energy demand (DSM)
+bev_dsm,--,"{true, false}",Add the option for battery electric vehicles (BEV) to participate in demand-side management (DSM)
+
 bev_availability,--,float,The percentage for battery electric vehicles (BEV) that are able to do demand side management (DSM)
 bev_energy,--,float,The average size of battery electric vehicles (BEV) in MWh
 bev_charge_efficiency,--,float,Battery electric vehicles (BEV) charge and discharge efficiency
@@ -33,14 +35,17 @@ agriculture_machinery_electric_efficiency,--,float,The efficiency of oil-powered
 MWh_MeOH_per_MWh_H2,LHV,float,"The energy amount of the produced methanol per energy amount of hydrogen. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, page 64."
 MWh_MeOH_per_tCO2,LHV,float,"The energy amount of the produced methanol per ton of CO2. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, page 64."
 MWh_MeOH_per_MWh_e,LHV,float,"The energy amount of the produced methanol per energy amount of electricity. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, page 64."
-shipping_hydrogen_liquefaction,--,"{true, false}",Consider whether to include liquefaction costs for shipping H2 demand.
+shipping_hydrogen_liquefaction,--,"{true, false}",Whether to include liquefaction costs for hydrogen demand in shipping.
+
 shipping_hydrogen_share,--,Dictionary with planning horizons as keys.,The share of ships powered by hydrogen in a given year
 shipping_methanol_share,--,Dictionary with planning horizons as keys.,The share of ships powered by methanol in a given year
 shipping_oil_share,--,Dictionary with planning horizons as keys.,The share of ships powered by oil in a given year
-shipping_methanol_efficiency,--,float,"The efficiency of methanol-powered ships in the conversion of methanol to meet shipping needs (propulsion). The efficiency increase from oil can be 10-15% higher according to the `IEA <https://www.iea-amf.org/app/webroot/files/file/Annex%20Reports/AMF_Annex_56.pdf>`_,"
+shipping_methanol_efficiency,--,float,"The efficiency of methanol-powered ships in the conversion of methanol to meet shipping needs (propulsion). The efficiency increase from oil can be 10-15% higher according to the `IEA <https://www.iea-amf.org/app/webroot/files/file/Annex%20Reports/AMF_Annex_56.pdf>`_"
+
 shipping_oil_efficiency,--,float,The efficiency of oil-powered ships in the conversion of oil to meet shipping needs (propulsion). Base value derived from 2011
-aviation_demand_factor,--,float,The proportion of demand for aviation compared to today's
-HVC_demand_factor,--,float,The proportion of demand for high-value chemicals compared to today's
+aviation_demand_factor,--,float,The proportion of demand for aviation compared to today's consumption
+HVC_demand_factor,--,float,The proportion of demand for high-value chemicals compared to today's consumption
+
 time_dep_hp_cop,--,"{true, false}",Consider the time dependent coefficient of performance (COP) of the heat pump
 heat_pump_sink_T,°C,float,The temperature heat sink used in heat pumps based on DTU / large area radiators. The value is conservatively high to cover hot water and space heating in poorly-insulated buildings
 reduce_space_heat_exogenously,--,"{true, false}",Influence on space heating demand by a certain factor (applied before losses in district heating).
@@ -82,27 +87,32 @@ regional_co2_sequestration_potential,,,
 -- years_of_storage,years,float,The years until potential exhausted at optimised annual rate
 co2_sequestration_potential,MtCO2/a,float,The potential of sequestering CO2 in Europe per year
 co2_sequestration_cost,currency/tCO2,float,The cost of sequestering a ton of CO2
-co2_spatial,--,"{true, false}","„Add option to spatially resolve carrier representing stored carbon dioxide. This allows for more detailed modelling of CCUTS, e.g. regarding the capturing of industrial process emissions, usage as feedstock for electrofuels, transport of carbon dioxide, and geological sequestration sites.“"
-co2network,--,"{true, false}",Add option for planning a new carbon dioxide network
+co2_spatial,--,"{true, false}","Add option to spatially resolve carrier representing stored carbon dioxide. This allows for more detailed modelling of CCUTS, e.g. regarding the capturing of industrial process emissions, usage as feedstock for electrofuels, transport of carbon dioxide, and geological sequestration sites."
+
+co2network,--,"{true, false}",Add option for planning a new carbon dioxide transmission network
+
 cc_fraction,--,float,The default fraction of CO2 captured with post-combustion capture
 hydrogen_underground_storage,--,"{true, false}",Add options for storing hydrogen underground. Storage potential depends regionally.
-hydrogen_underground_storage_locations,,"{onshore, nearshore, offshore}","„The location where hydrogen underground storage can be located. Onshore, nearshore, offshore means it must be located more than 50 km away from the sea, within 50 km of the sea, or within the sea itself respectively.“"
+hydrogen_underground_storage_locations,,"{onshore, nearshore, offshore}","The location where hydrogen underground storage can be located. Onshore, nearshore, offshore means it must be located more than 50 km away from the sea, within 50 km of the sea, or within the sea itself respectively."
+
 ammonia,--,"{true, false, regional}","Add ammonia as a carrrier. It can be either true (copperplated NH3), false (no NH3 carrier) or ""regional"" (regionalised NH3 without network)"
-min_part_load_fischer_tropsch,per unit of p_nom ,float,The minimum unit dispatch (p_min_pu) for the Fischer-Tropsch process
-min_part_load_methanolisation,per unit of p_nom ,float,The minimum unit dispatch (p_min_pu) for the methanolisation process
+min_part_load_fischer_tropsch,per unit of p_nom ,float,The minimum unit dispatch (``p_min_pu``) for the Fischer-Tropsch process
+min_part_load_methanolisation,per unit of p_nom ,float,The minimum unit dispatch (``p_min_pu``) for the methanolisation process
+
 use_fischer_tropsch_waste_heat,--,"{true, false}",Add option for using waste heat of Fischer Tropsch in district heating networks
 use_fuel_cell_waste_heat,--,"{true, false}",Add option for using waste heat of fuel cells in district heating networks
 use_electrolysis_waste_heat,--,"{true, false}",Add option for using waste heat of electrolysis in district heating networks
-electricity_distribution_grid,--,"{true, false}",Add a electricity distribution grid
+electricity_distribution_grid,--,"{true, false}",Add a simplified representation of the exchange capacity between transmission and distribution grid level through a link.
 electricity_distribution_grid_cost_factor,,,Multiplies the investment cost of the electricity distribution grid in data/costs.csv
 electricity_grid_connection,--,"{true, false}",Add the cost of electricity grid connection for onshore wind and solar
 H2_network,--,"{true, false}",Add option for new hydrogen pipelines
 gas_network,--,"{true, false}","Add existing natural gas infrastructure, incl. LNG terminals, production and entry-points. The existing gas network is added with a lossless transport model. A length-weighted `k-edge augmentation algorithm   <https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation>`_   can be run to add new candidate gas pipelines such that all regions of the model can be connected to the gas network. When activated, all the gas demands are regionally disaggregated as well."
-H2_retrofit,--,"{true, false}","Add option for retrofiting existing pipelines to transport hydrogen. The reasoning is in accordance with the `hydrogen backbone strategy (April, 2020) p.15 <https://gasforclimate2050.eu/wp-content/uploads/2020/07/2020_European-Hydrogen-Backbone_Report.pdf>`_. 60% of original natural gas capacity could be used in cost-optimal case as H2 capacity."
-H2_retrofit_capacity_per_CH4,--,float,The ratio for H2 capacity per original CH4 capacity of retrofitted pipelines
+H2_retrofit,--,"{true, false}","Add option for retrofiting existing pipelines to transport hydrogen. "
+H2_retrofit_capacity_per_CH4,--,float,"The ratio for H2 capacity per original CH4 capacity of retrofitted pipelines. The `European Hydrogen Backbone (April, 2020) p.15 <https://gasforclimate2050.eu/wp-content/uploads/2020/07/2020_European-Hydrogen-Backbone_Report.pdf>`_ 60% of original natural gas capacity could be used in cost-optimal case as H2 capacity."
 gas_network_connectivity_upgrade ,--,float,The number of desired edge connectivity (k) in the length-weighted `k-edge augmentation algorithm <https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation>`_ used for the gas network
 gas_distribution_grid,--,"{true, false}",Add a gas distribution grid
-gas_distribution_grid_cost_factor,,,Multiplies the investment cost of the gas distribution grid in data/costs.csv
+gas_distribution_grid_cost_factor,,,Multiplier for the investment cost of the gas distribution grid
+
 biomass_spatial,--,"{true, false}",Add option for resolving biomass demand regionally
 biomass_transport,--,"{true, false}",Add option for transporting solid biomass between nodes
 conventional_generation,,,Add a more detailed description of conventional carriers. Any power generation requires the consumption of fuel from nodes representing that fuel.
diff --git a/doc/configtables/solar-thermal.csv b/doc/configtables/solar-thermal.csv
index 7cc8dfc7..4575ae0d 100644
--- a/doc/configtables/solar-thermal.csv
+++ b/doc/configtables/solar-thermal.csv
@@ -1,5 +1,6 @@
 ,Unit,Values,Description
 clearsky_model ,--,"{‘simple’, ‘enhanced’}",Type of clearsky model for diffuse irradiation
 orientation ,--,"{units of degrees, ‘latitude_optimal’}",Panel orientation with slope and azimuth
--- azimuth,float,units of degrees,The angle between the north and the sun with panels on the local horizon
+-- azimuth,float,units of degrees,The angle between the North and the sun with panels on the local horizon
+
 -- slope,float,units of degrees,The angle between the ground and the panels

From 4ecc7ae43904b392059fcc60ce56b38ef6f3b78a Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 12 Jul 2023 10:42:46 +0000
Subject: [PATCH 159/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 doc/configtables/energy.csv | 1 -
 1 file changed, 1 deletion(-)

diff --git a/doc/configtables/energy.csv b/doc/configtables/energy.csv
index ff5e48cd..8718d75e 100644
--- a/doc/configtables/energy.csv
+++ b/doc/configtables/energy.csv
@@ -5,4 +5,3 @@ base_emissions_year ,--,"YYYY; e.g. 1990","The base year for the sector emission
 eurostat_report_year ,--,"{2016,2017,2018}","The publication year of the Eurostat report. 2016 includes Bosnia and Herzegovina, 2017 does not"
 
 emissions ,--,"{CO2, All greenhouse gases - (CO2 equivalent)}","Specify which sectoral emissions are taken into account. Data derived from EEA. Currently only CO2 is implemented."
-

From 8c4299ee103bbfd8bf76ebe1e08938cb93fafae9 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Wed, 12 Jul 2023 12:42:54 +0200
Subject: [PATCH 160/176] Delete .~lock.sector.csv#

---
 doc/configtables/.~lock.sector.csv# | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 doc/configtables/.~lock.sector.csv#

diff --git a/doc/configtables/.~lock.sector.csv# b/doc/configtables/.~lock.sector.csv#
deleted file mode 100644
index 24f46f24..00000000
--- a/doc/configtables/.~lock.sector.csv#
+++ /dev/null
@@ -1 +0,0 @@
-,DESKTOP-2RNCH2B/kunde,,05.07.2023 01:09,file:///C:/Users/kunde/AppData/Roaming/LibreOffice/4;

From 30a81a559f499aa4f1e29682269c838f3df3bd9b Mon Sep 17 00:00:00 2001
From: euronion <42553970+euronion@users.noreply.github.com>
Date: Thu, 13 Jul 2023 08:22:22 +0200
Subject: [PATCH 161/176] Add test and option to disable online retrieve rules

---
 Snakefile                  |  5 ++++-
 config/config.default.yaml |  1 +
 rules/common.smk           | 16 ++++++++++++++++
 3 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/Snakefile b/Snakefile
index 27ed4dfb..048bae28 100644
--- a/Snakefile
+++ b/Snakefile
@@ -48,7 +48,10 @@ wildcard_constraints:
 
 include: "rules/common.smk"
 include: "rules/collect.smk"
-include: "rules/retrieve.smk"
+if (config["enable"].get("retrieve","auto") == "auto" and has_internet_access())or config["enable"]["retrieve"] is True:
+    include: "rules/retrieve.smk"
+else:
+    print("Datafile downloads disabled in config[retrieve] or no internet access.")
 include: "rules/build_electricity.smk"
 include: "rules/build_sector.smk"
 include: "rules/solve_electricity.smk"
diff --git a/config/config.default.yaml b/config/config.default.yaml
index c4b2620f..2e2c15dd 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -67,6 +67,7 @@ snapshots:
   inclusive: 'left' # include start, not end
 
 enable:
+  retrieve: auto
   prepare_links_p_nom: false
   retrieve_databundle: true
   retrieve_sector_databundle: true
diff --git a/rules/common.smk b/rules/common.smk
index 8f0c7cbb..3fc2c721 100644
--- a/rules/common.smk
+++ b/rules/common.smk
@@ -23,6 +23,22 @@ def memory(w):
         return int(factor * (10000 + 195 * int(w.clusters)))
 
 
+# Check if the workflow has access to the internet by trying to access the HEAD of specified url
+def has_internet_access(url="www.zenodo.org") -> bool:
+    import http.client as http_client
+
+    # based on answer and comments from
+    # https://stackoverflow.com/a/29854274/11318472
+    conn = http_client.HTTPConnection(url, timeout=5)  # need access to zenodo anyway
+    try:
+        conn.request("HEAD", "/")
+        return True
+    except:
+        return False
+    finally:
+        conn.close()
+
+
 def input_eurostat(w):
     # 2016 includes BA, 2017 does not
     report_year = config["energy"]["eurostat_report_year"]

From 3582c27e2ecf59bf6f30c8e006316abb34e895f3 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Thu, 13 Jul 2023 10:31:40 +0200
Subject: [PATCH 162/176] Update doc/configuration.rst

---
 doc/configuration.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/configuration.rst b/doc/configuration.rst
index 7e911ef9..cb8127a9 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -64,7 +64,7 @@ The ``run`` section is used for running and storing scenarios with different con
    :file: configtables/foresight.csv
 
 .. note::
-    if you use myopic or perfect foresight, The investment years in
+    If you use myopic or perfect foresight, the planning horizon in
     :ref:`planning_horizons` in scenario has to be set.
 
 .. _scenario:

From 56fd99a10ec060e5457ffb4d4cd4242f7a415300 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Thu, 13 Jul 2023 11:45:46 +0200
Subject: [PATCH 163/176] Apply three Fabian Neumann suggestions from code
 review

Co-authored-by: Fabian Neumann <fabian.neumann@outlook.de>
---
 doc/configtables/sector.csv | 8 +++++---
 doc/configuration.rst       | 2 +-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/doc/configtables/sector.csv b/doc/configtables/sector.csv
index 7f7efed4..d403e676 100644
--- a/doc/configtables/sector.csv
+++ b/doc/configtables/sector.csv
@@ -7,8 +7,9 @@ cluster_heat_buses,--,"{true, false}",Cluster residential and service heat buses
 
 bev_dsm_restriction_value,--,float,Adds a lower state of charge (SOC) limit for battery electric vehicles (BEV) to manage its own energy demand (DSM). Located in `build_transport_demand.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/build_transport_demand.py>`_. Set to 0 for no restriction on BEV DSM
 bev_dsm_restriction_time,--,float,Time at which SOC of BEV has to be dsm_restriction_value
-transport_heating_deadband_upper,°C,float,"The minimum temperature in the vehicle. At lower temperatures, the energy required for heating in the vehicle increases."
-transport_heating_deadband_lower,°C,float,"The maximum temperature in the vehicle. At higher temperatures, the energy required for cooling in the vehicle increases."
+transport_heating_deadband_upper,°C,float,"The maximum temperature in the vehicle. At higher temperatures, the energy required for cooling in the vehicle increases."
+transport_heating_deadband_lower,°C,float,"The minimum temperature in the vehicle. At lower temperatures, the energy required for heating in the vehicle increases."
+
 ICE_lower_degree_factor,--,float,Percentage increase in energy demand in internal combustion engine (ICE) for each degree difference between the cold environment and the minimum temperature.
 ICE_upper_degree_factor,--,float,Percentage increase in energy demand in internal combustion engine (ICE) for each degree difference between the hot environment and the maximum temperature.
 EV_lower_degree_factor,--,float,Percentage increase in energy demand in electric vehicles (EV) for each degree difference between the cold environment and the minimum temperature.
@@ -103,7 +104,8 @@ use_fischer_tropsch_waste_heat,--,"{true, false}",Add option for using waste hea
 use_fuel_cell_waste_heat,--,"{true, false}",Add option for using waste heat of fuel cells in district heating networks
 use_electrolysis_waste_heat,--,"{true, false}",Add option for using waste heat of electrolysis in district heating networks
 electricity_distribution_grid,--,"{true, false}",Add a simplified representation of the exchange capacity between transmission and distribution grid level through a link.
-electricity_distribution_grid_cost_factor,,,Multiplies the investment cost of the electricity distribution grid in data/costs.csv
+electricity_distribution_grid_cost_factor,,,Multiplies the investment cost of the electricity distribution grid
+
 electricity_grid_connection,--,"{true, false}",Add the cost of electricity grid connection for onshore wind and solar
 H2_network,--,"{true, false}",Add option for new hydrogen pipelines
 gas_network,--,"{true, false}","Add existing natural gas infrastructure, incl. LNG terminals, production and entry-points. The existing gas network is added with a lossless transport model. A length-weighted `k-edge augmentation algorithm   <https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation>`_   can be run to add new candidate gas pipelines such that all regions of the model can be connected to the gas network. When activated, all the gas demands are regionally disaggregated as well."
diff --git a/doc/configuration.rst b/doc/configuration.rst
index cb8127a9..f8d97c7b 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -473,7 +473,7 @@ The list of available biomass is given by the category in `ENSPRESO_BIOMASS <htt
 =======================
 
 .. note::
-   Only used for sector-coupling studies. The value for grouping years are only used in myopic or perfect scenarios.
+   Only used for sector-coupling studies. The value for grouping years are only used in myopic or perfect foresight scenarios.
 
 .. literalinclude:: ../config/config.default.yaml
    :language: yaml

From 604e4f2a834f6e4e5b48c79c04c628b18f7255dd Mon Sep 17 00:00:00 2001
From: Win_Surface_virio_andreyana <arsyan.virio@gmail.com>
Date: Thu, 13 Jul 2023 12:52:18 +0200
Subject: [PATCH 164/176] add description on technology_mapping

---
 doc/configtables/electricity.csv | 17 ++++++++++-------
 doc/configuration.rst            |  3 ---
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv
index 828c34df..eb087456 100644
--- a/doc/configtables/electricity.csv
+++ b/doc/configtables/electricity.csv
@@ -5,7 +5,7 @@ co2limit,:math:`t_{CO_2-eq}/a`,float,Cap on total annual system carbon dioxide e
 co2base,:math:`t_{CO_2-eq}/a`,float,Reference value of total annual system carbon dioxide emissions if relative emission reduction target is specified in ``{opts}`` wildcard.
 agg_p_nom_limits,file,path,Reference to ``.csv`` file specifying per carrier generator nominal capacity constraints for individual countries if ``'CCL'`` is in ``{opts}`` wildcard. Defaults to ``data/agg_p_nom_minmax.csv``.
 operational_reserve,,,Settings for reserve requirements following `GenX <https://genxproject.github.io/GenX/dev/core/#Reserves>`_
-
+,,,
 -- activate,bool,true or false,Whether to take operational reserve requirements into account during optimisation
 -- epsilon_load,--,float,share of total load
 -- epsilon_vres,--,float,share of total renewable supply
@@ -19,15 +19,18 @@ extendable_carriers,,,
 -- Store,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity.
 -- Link,--,Any subset of {'H2 pipeline'},Adds extendable links (H2 pipelines only) at every connection where there are lines or HVDC links without capacity limits and with zero initial capacity. Hydrogen pipelines require hydrogen storage to be modelled as ``Store``.
 powerplants_filter,--,"use `pandas.query <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.query.html>`_ strings here, e.g. ``Country not in ['Germany']``",Filter query for the default powerplant database.
-
+,,,
 custom_powerplants,--,"use `pandas.query <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.query.html>`_ strings here, e.g. ``Country in ['Germany']``",Filter query for the custom powerplant database.
-
+,,,
 conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to include in the model from ``resources/powerplants.csv``. If an included carrier is also listed in ``extendable_carriers``, the capacity is taken as a lower bound."
-
+,,,
 renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, hydro}",List of renewable generators to include in the model.
 estimate_renewable_capacities,,,
 -- enable,,bool,Activate routine to estimate renewable capacities
--- from_opsd,--,bool,Add capacities from OPSD data
--- year,--,bool,Renewable capacities are based on existing capacities reported by IRENA for the specified year
+-- from_opsd,--,bool,Add renewable capacities from `OPSD database <https://data.open-power-system-data.org/renewable_power_plants/2020-08-25>`_. The value is depreciated but still can be used.   
+-- year,--,bool,Renewable capacities are based on existing capacities reported by IRENA (IRENASTAT) for the specified year
 -- expansion_limit,--,float or false,"Artificially limit maximum IRENA capacities to a factor.  For example, an ``expansion_limit: 1.1`` means 110% of capacities . If false are chosen, the estimated renewable potentials determine by the workflow are used."
--- technology_mapping,,,Mapping between powerplantmatching and PyPSA-Eur technology names
+-- technology_mapping,,,Mapping between PyPSA-Eur and powerplantmatching technology names
+-- -- Offshore,--,"Any subset of {offwind-ac, offwind-dc}","List of PyPSA-Eur carriers that is considered as (IRENA, OPSD) onshore technology."
+-- -- Offshore,--,{onwind},"List of PyPSA-Eur carriers that is considered as (IRENA, OPSD) offshore technology."
+-- -- PV,--,{solar},"List of PyPSA-Eur carriers that is considered as (IRENA, OPSD) PV technology."
diff --git a/doc/configuration.rst b/doc/configuration.rst
index cb8127a9..5e288a7f 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -191,9 +191,6 @@ Switches for some rules and optional features.
    :widths: 25,7,22,30
    :file: configtables/electricity.csv
 
-.. note::
-   Wind is the Fueltype in powerplantmatching, onwind, offwind-{ac,dc} the carrier in PyPSA-Eur
-
 .. _atlite_cf:
 
 ``atlite``

From a03574ba14fa69d8199455925d8d46cfd3f91c3f Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 13 Jul 2023 10:55:56 +0000
Subject: [PATCH 165/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 doc/configtables/electricity.csv | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv
index eb087456..4c04fee6 100644
--- a/doc/configtables/electricity.csv
+++ b/doc/configtables/electricity.csv
@@ -27,7 +27,7 @@ conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite
 renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, hydro}",List of renewable generators to include in the model.
 estimate_renewable_capacities,,,
 -- enable,,bool,Activate routine to estimate renewable capacities
--- from_opsd,--,bool,Add renewable capacities from `OPSD database <https://data.open-power-system-data.org/renewable_power_plants/2020-08-25>`_. The value is depreciated but still can be used.   
+-- from_opsd,--,bool,Add renewable capacities from `OPSD database <https://data.open-power-system-data.org/renewable_power_plants/2020-08-25>`_. The value is depreciated but still can be used.
 -- year,--,bool,Renewable capacities are based on existing capacities reported by IRENA (IRENASTAT) for the specified year
 -- expansion_limit,--,float or false,"Artificially limit maximum IRENA capacities to a factor.  For example, an ``expansion_limit: 1.1`` means 110% of capacities . If false are chosen, the estimated renewable potentials determine by the workflow are used."
 -- technology_mapping,,,Mapping between PyPSA-Eur and powerplantmatching technology names

From 730b32059696a9b19709b98243bba0c537d26587 Mon Sep 17 00:00:00 2001
From: euronion <42553970+euronion@users.noreply.github.com>
Date: Fri, 14 Jul 2023 08:53:39 +0200
Subject: [PATCH 166/176] Move logic to retrieve.smk

---
 Snakefile          |  5 +--
 rules/retrieve.smk | 93 +++++++++++++++++++++++++++-------------------
 2 files changed, 56 insertions(+), 42 deletions(-)

diff --git a/Snakefile b/Snakefile
index 048bae28..27ed4dfb 100644
--- a/Snakefile
+++ b/Snakefile
@@ -48,10 +48,7 @@ wildcard_constraints:
 
 include: "rules/common.smk"
 include: "rules/collect.smk"
-if (config["enable"].get("retrieve","auto") == "auto" and has_internet_access())or config["enable"]["retrieve"] is True:
-    include: "rules/retrieve.smk"
-else:
-    print("Datafile downloads disabled in config[retrieve] or no internet access.")
+include: "rules/retrieve.smk"
 include: "rules/build_electricity.smk"
 include: "rules/build_sector.smk"
 include: "rules/solve_electricity.smk"
diff --git a/rules/retrieve.smk b/rules/retrieve.smk
index 0a96406a..a253ec0c 100644
--- a/rules/retrieve.smk
+++ b/rules/retrieve.smk
@@ -2,7 +2,14 @@
 #
 # SPDX-License-Identifier: MIT
 
-if config["enable"].get("retrieve_databundle", True):
+if config["enable"].get("retrieve", "auto") == "auto":
+    config["enable"]["retrieve"] = has_internet_access()
+
+if config["enable"]["retrieve"] is False:
+    print("Datafile downloads disabled in config[retrieve] or no internet access.")
+
+
+if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", True):
     datafiles = [
         "ch_cantons.csv",
         "je-e-21.03.02.xls",
@@ -32,7 +39,7 @@ if config["enable"].get("retrieve_databundle", True):
             "../scripts/retrieve_databundle.py"
 
 
-if config["enable"].get("retrieve_cutout", True):
+if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True):
 
     rule retrieve_cutout:
         input:
@@ -51,7 +58,7 @@ if config["enable"].get("retrieve_cutout", True):
             move(input[0], output[0])
 
 
-if config["enable"].get("retrieve_cost_data", True):
+if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", True):
 
     rule retrieve_cost_data:
         input:
@@ -73,7 +80,9 @@ if config["enable"].get("retrieve_cost_data", True):
             move(input[0], output[0])
 
 
-if config["enable"].get("retrieve_natura_raster", True):
+if config["enable"]["retrieve"] and config["enable"].get(
+    "retrieve_natura_raster", True
+):
 
     rule retrieve_natura_raster:
         input:
@@ -93,7 +102,9 @@ if config["enable"].get("retrieve_natura_raster", True):
             move(input[0], output[0])
 
 
-if config["enable"].get("retrieve_sector_databundle", True):
+if config["enable"]["retrieve"] and config["enable"].get(
+    "retrieve_sector_databundle", True
+):
     datafiles = [
         "data/eea/UNFCCC_v23.csv",
         "data/switzerland-sfoe/switzerland-new_format.csv",
@@ -120,7 +131,9 @@ if config["enable"].get("retrieve_sector_databundle", True):
             "../scripts/retrieve_sector_databundle.py"
 
 
-if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
+if config["enable"]["retrieve"] and (
+    config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]
+):
     datafiles = [
         "IGGIELGN_LNGs.geojson",
         "IGGIELGN_BorderPoints.geojson",
@@ -140,37 +153,41 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
             "../scripts/retrieve_gas_infrastructure_data.py"
 
 
-rule retrieve_electricity_demand:
-    input:
-        HTTP.remote(
-            "data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv",
-            keep_local=True,
-            static=True,
-        ),
-    output:
-        "data/load_raw.csv",
-    log:
-        LOGS + "retrieve_electricity_demand.log",
-    resources:
-        mem_mb=5000,
-    retries: 2
-    run:
-        move(input[0], output[0])
+if config["enable"]["retrieve"]:
+
+    rule retrieve_electricity_demand:
+        input:
+            HTTP.remote(
+                "data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv",
+                keep_local=True,
+                static=True,
+            ),
+        output:
+            "data/load_raw.csv",
+        log:
+            LOGS + "retrieve_electricity_demand.log",
+        resources:
+            mem_mb=5000,
+        retries: 2
+        run:
+            move(input[0], output[0])
 
 
-rule retrieve_ship_raster:
-    input:
-        HTTP.remote(
-            "https://zenodo.org/record/6953563/files/shipdensity_global.zip",
-            keep_local=True,
-            static=True,
-        ),
-    output:
-        "data/shipdensity_global.zip",
-    log:
-        LOGS + "retrieve_ship_raster.log",
-    resources:
-        mem_mb=5000,
-    retries: 2
-    run:
-        move(input[0], output[0])
+if config["enable"]["retrieve"]:
+
+    rule retrieve_ship_raster:
+        input:
+            HTTP.remote(
+                "https://zenodo.org/record/6953563/files/shipdensity_global.zip",
+                keep_local=True,
+                static=True,
+            ),
+        output:
+            "data/shipdensity_global.zip",
+        log:
+            LOGS + "retrieve_ship_raster.log",
+        resources:
+            mem_mb=5000,
+        retries: 2
+        run:
+            move(input[0], output[0])

From 23b154de36c7251fd1c443c46239548e58500d82 Mon Sep 17 00:00:00 2001
From: virio-andreyana <arsyan.virio@gmail.com>
Date: Fri, 14 Jul 2023 12:25:23 +0200
Subject: [PATCH 167/176] adjust config table spacing

---
 doc/configtables/industry.csv |  26 ++++-----
 doc/configtables/sector.csv   | 106 +++++++++++++++++-----------------
 doc/configuration.rst         |  60 +++++++++----------
 doc/tutorial.rst              |  16 ++---
 doc/tutorial_sector.rst       |  12 ++--
 5 files changed, 110 insertions(+), 110 deletions(-)

diff --git a/doc/configtables/industry.csv b/doc/configtables/industry.csv
index 358da5c8..fc1b3f0f 100644
--- a/doc/configtables/industry.csv
+++ b/doc/configtables/industry.csv
@@ -1,30 +1,30 @@
 ,Unit,Values,Description
 St_primary_fraction,--,Dictionary with planning horizons as keys.,The fraction of steel produced via primary route versus secondary route (scrap+EAF). Current fraction is 0.6
 DRI_fraction,--,Dictionary with planning horizons as keys.,The fraction of the primary route DRI + EAF
-
+,,,
 H2_DRI,--,float,The hydrogen consumption in Direct Reduced Iron (DRI)  Mwh_H2 LHV/ton_Steel from 51kgH2/tSt in `Vogl et al (2018) <https://doi.org/10.1016/j.jclepro.2018.08.279>`_
 elec_DRI,MWh/tSt,float,The electricity consumed in Direct Reduced Iron (DRI) shaft. From `HYBRIT brochure <https://ssabwebsitecdn.azureedge.net/-/media/hybrit/files/hybrit_brochure.pdf>`_
 Al_primary_fraction,--,Dictionary with planning horizons as keys.,The fraction of aluminium produced via the primary route versus scrap. Current fraction is 0.4
 MWh_NH3_per_tNH3,LHV,float,The energy amount per ton of ammonia.
 MWh_CH4_per_tNH3_SMR,--,float,The energy amount of methane needed to produce a ton of ammonia using steam methane reforming (SMR). Value derived from 2012's demand from `Center for European Policy Studies (2008) <https://ec.europa.eu/docsroom/documents/4165/attachments/1/translations/en/renditions/pdf>`_
 MWh_elec_per_tNH3_SMR,--,float,"The energy amount of electricity needed to produce a ton of ammonia using steam methane reforming (SMR). same source, assuming 94-6% split methane-elec of total energy demand 11.5 MWh/tNH3"
-MWh_H2_per_tNH3_electrolysis,--,float,"The energy amount of hydrogen needed to produce a ton of ammonia using Haber–Bosch process. From  `Wang et al (2018) <https://doi.org/10.1016/j.joule.2018.04.017>`_, Base value assumed around 0.197 tH2/tHN3 (>3/17 since some H2 lost and used for energy)"
-MWh_elec_per_tNH3_electrolysis,--,float,"The energy amount of electricity needed to produce a ton of ammonia using Haber–Bosch process. From  `Wang et al (2018) <https://doi.org/10.1016/j.joule.2018.04.017>`_, Table 13 (air separation and HB)"
-MWh_NH3_per_MWh_H2_cracker,--,float,The energy amount of amonia needed to produce an energy amount hydrogen using ammonia cracker
+Mwh_H2_per_tNH3 _electrolysis,--,float,"The energy amount of hydrogen needed to produce a ton of ammonia using Haber–Bosch process. From  `Wang et al (2018) <https://doi.org/10.1016/j.joule.2018.04.017>`_, Base value assumed around 0.197 tH2/tHN3 (>3/17 since some H2 lost and used for energy)"
+Mwh_elec_per_tNH3 _electrolysis,--,float,"The energy amount of electricity needed to produce a ton of ammonia using Haber–Bosch process. From  `Wang et al (2018) <https://doi.org/10.1016/j.joule.2018.04.017>`_, Table 13 (air separation and HB)"
+Mwh_NH3_per_MWh _H2_cracker,--,float,The energy amount of amonia needed to produce an energy amount hydrogen using ammonia cracker
 NH3_process_emissions,MtCO2/a,float,The emission of ammonia production from steam methane reforming (SMR). From UNFCCC for 2015 for EU28
-petrochemical_process_emissions,MtCO2/a,float,The emission of petrochemical production. From UNFCCC for 2015 for EU28
+petrochemical_process _emissions,MtCO2/a,float,The emission of petrochemical production. From UNFCCC for 2015 for EU28
 HVC_primary_fraction,--,float,The fraction of high value chemicals (HVC) produced via primary route
-HVC_mechanical_recycling_fraction,--,float,The fraction of high value chemicals (HVC) produced using mechanical recycling
-HVC_chemical_recycling_fraction,--,float,The fraction of high value chemicals (HVC) produced using chemical recycling
-
+HVC_mechanical_recycling _fraction,--,float,The fraction of high value chemicals (HVC) produced using mechanical recycling
+HVC_chemical_recycling _fraction,--,float,The fraction of high value chemicals (HVC) produced using chemical recycling
+,,,
 HVC_production_today,MtHVC/a,float,"The amount of high value chemicals (HVC) produced. This includes ethylene, propylene and BTX. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, Figure 16, page 107"
-MWh_elec_per_tHVC_mechanical_recycling,MWh/tHVC,float,"The energy amount of electricity needed to produce a ton of high value chemical (HVC) using mechanical recycling. From SI of `Meys et al (2020) <https://doi.org/10.1016/j.resconrec.2020.105010>`_, Table S5, for HDPE, PP, PS, PET. LDPE would be 0.756."
-MWh_elec_per_tHVC_chemical_recycling,MWh/tHVC,float,"The energy amount of electricity needed to produce a ton of high value chemical (HVC) using chemical recycling. The default value is based on pyrolysis and electric steam cracking. From `Material Economics (2019) <https://materialeconomics.com/latest-updates/industrial-transformation-2050>`_, page 125"
-
-chlorine_production_today,MtCl/a,float,"The amount of chlorine produced. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, Table 7, page 43"
+Mwh_elec_per_tHVC _mechanical_recycling,MWh/tHVC,float,"The energy amount of electricity needed to produce a ton of high value chemical (HVC) using mechanical recycling. From SI of `Meys et al (2020) <https://doi.org/10.1016/j.resconrec.2020.105010>`_, Table S5, for HDPE, PP, PS, PET. LDPE would be 0.756."
+Mwh_elec_per_tHVC _chemical_recycling,MWh/tHVC,float,"The energy amount of electricity needed to produce a ton of high value chemical (HVC) using chemical recycling. The default value is based on pyrolysis and electric steam cracking. From `Material Economics (2019) <https://materialeconomics.com/latest-updates/industrial-transformation-2050>`_, page 125"
+,,,
+chlorine_production _today,MtCl/a,float,"The amount of chlorine produced. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, Table 7, page 43"
 MWh_elec_per_tCl,MWh/tCl,float,"The energy amount of electricity needed to produce a ton of chlorine. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, Table 6 page 43"
 MWh_H2_per_tCl,MWhH2/tCl,float,"The energy amount of hydrogen needed to produce a ton of chlorine. The value is negative since hydrogen produced in chloralkali process. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, page 43"
-methanol_production_today,MtMeOH/a,float,"The amount of methanol produced. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, page 62"
+methanol_production _today,MtMeOH/a,float,"The amount of methanol produced. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, page 62"
 MWh_elec_per_tMeOH,MWh/tMeOH,float,"The energy amount of electricity needed to produce a ton of methanol. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, Table 14, page 65"
 MWh_CH4_per_tMeOH,MWhCH4/tMeOH,float,"The energy amount of methane needed to produce a ton of methanol. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, Table 14, page 65"
 hotmaps_locate_missing,--,"{true,false}",Locate industrial sites without valid locations based on city and countries.
diff --git a/doc/configtables/sector.csv b/doc/configtables/sector.csv
index d403e676..36b7dd98 100644
--- a/doc/configtables/sector.csv
+++ b/doc/configtables/sector.csv
@@ -2,55 +2,55 @@
 district_heating,--,,`prepare_sector_network.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/prepare_sector_network.py>`_
 -- potential,--,float,maximum fraction of urban demand which can be supplied by district heating
 -- progress,--,Dictionary with planning horizons as keys., Increase of today's district heating demand to potential maximum district heating share. Progress = 0 means today's district heating share. Progress = 1 means maximum fraction of urban demand is supplied by district heating
--- district_heating_loss,--,float,Percentage increase in district heat demand in urban central due to heat losses
+-- district_heating_loss,--,float,Share increase in district heat demand in urban central due to heat losses
 cluster_heat_buses,--,"{true, false}",Cluster residential and service heat buses in `prepare_sector_network.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/prepare_sector_network.py>`_  to one to save memory.
-
-bev_dsm_restriction_value,--,float,Adds a lower state of charge (SOC) limit for battery electric vehicles (BEV) to manage its own energy demand (DSM). Located in `build_transport_demand.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/build_transport_demand.py>`_. Set to 0 for no restriction on BEV DSM
-bev_dsm_restriction_time,--,float,Time at which SOC of BEV has to be dsm_restriction_value
-transport_heating_deadband_upper,°C,float,"The maximum temperature in the vehicle. At higher temperatures, the energy required for cooling in the vehicle increases."
-transport_heating_deadband_lower,°C,float,"The minimum temperature in the vehicle. At lower temperatures, the energy required for heating in the vehicle increases."
-
-ICE_lower_degree_factor,--,float,Percentage increase in energy demand in internal combustion engine (ICE) for each degree difference between the cold environment and the minimum temperature.
-ICE_upper_degree_factor,--,float,Percentage increase in energy demand in internal combustion engine (ICE) for each degree difference between the hot environment and the maximum temperature.
-EV_lower_degree_factor,--,float,Percentage increase in energy demand in electric vehicles (EV) for each degree difference between the cold environment and the minimum temperature.
-EV_upper_degree_factor,--,float,Percentage increase in energy demand in electric vehicles (EV) for each degree difference between the hot environment and the maximum temperature.
+,,,
+bev_dsm_restriction _value,--,float,Adds a lower state of charge (SOC) limit for battery electric vehicles (BEV) to manage its own energy demand (DSM). Located in `build_transport_demand.py <https://github.com/PyPSA/pypsa-eur-sec/blob/master/scripts/build_transport_demand.py>`_. Set to 0 for no restriction on BEV DSM
+bev_dsm_restriction _time,--,float,Time at which SOC of BEV has to be dsm_restriction_value
+transport_heating _deadband_upper,°C,float,"The maximum temperature in the vehicle. At higher temperatures, the energy required for cooling in the vehicle increases."
+transport_heating _deadband_lower,°C,float,"The minimum temperature in the vehicle. At lower temperatures, the energy required for heating in the vehicle increases."
+,,,
+ICE_lower_degree_factor,--,float,Share increase in energy demand in internal combustion engine (ICE) for each degree difference between the cold environment and the minimum temperature.
+ICE_upper_degree_factor,--,float,Share increase in energy demand in internal combustion engine (ICE) for each degree difference between the hot environment and the maximum temperature.
+EV_lower_degree_factor,--,float,Share increase in energy demand in electric vehicles (EV) for each degree difference between the cold environment and the minimum temperature.
+EV_upper_degree_factor,--,float,Share increase in energy demand in electric vehicles (EV) for each degree difference between the hot environment and the maximum temperature.
 bev_dsm,--,"{true, false}",Add the option for battery electric vehicles (BEV) to participate in demand-side management (DSM)
-
-bev_availability,--,float,The percentage for battery electric vehicles (BEV) that are able to do demand side management (DSM)
+,,,
+bev_availability,--,float,The share for battery electric vehicles (BEV) that are able to do demand side management (DSM)
 bev_energy,--,float,The average size of battery electric vehicles (BEV) in MWh
 bev_charge_efficiency,--,float,Battery electric vehicles (BEV) charge and discharge efficiency
-bev_plug_to_wheel_efficiency,km/kWh,float,The distance battery electric vehicles (BEV) can travel in km per kWh of energy charge in battery.  Base value comes from `Tesla Model S <https://www.fueleconomy.gov/feg/>`_
+bev_plug_to_wheel _efficiency,km/kWh,float,The distance battery electric vehicles (BEV) can travel in km per kWh of energy charge in battery.  Base value comes from `Tesla Model S <https://www.fueleconomy.gov/feg/>`_
 bev_charge_rate,MWh,float,The power consumption for one electric vehicle (EV) in MWh. Value derived from 3-phase charger with 11 kW.
-bev_avail_max,--,float,The maximum percentage plugged-in availability for passenger electric vehicles.
-bev_avail_mean,--,float,The average percentage plugged-in availability for passenger electric vehicles.
+bev_avail_max,--,float,The maximum share plugged-in availability for passenger electric vehicles.
+bev_avail_mean,--,float,The average share plugged-in availability for passenger electric vehicles.
 v2g,--,"{true, false}",Allows feed-in to grid from EV battery
-land_transport_fuel_cell_share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses fuel cells in a given year
-land_transport_electric_share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses electric vehicles (EV) in a given year
-land_transport_ice_share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses internal combustion engines (ICE) in a given year. What is not EV or FCEV is oil-fuelled ICE.
-transport_fuel_cell_efficiency,--,float,The H2 conversion efficiencies of fuel cells in transport
-transport_internal_combustion_efficiency,--,float,The oil conversion efficiencies of internal combustion engine (ICE) in transport
-agriculture_machinery_electric_share,--,float,The percentage for agricultural machinery that uses electricity
-agriculture_machinery_oil_share,--,float,The percentage for agricultural machinery that uses oil
-agriculture_machinery_fuel_efficiency,--,float,The efficiency of electric-powered machinery in the conversion of electricity to meet agricultural needs.
-agriculture_machinery_electric_efficiency,--,float,The efficiency of oil-powered machinery in the conversion of oil to meet agricultural needs.
-MWh_MeOH_per_MWh_H2,LHV,float,"The energy amount of the produced methanol per energy amount of hydrogen. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, page 64."
+land_transport_fuel_cell _share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses fuel cells in a given year
+land_transport_electric _share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses electric vehicles (EV) in a given year
+land_transport_ice _share,--,Dictionary with planning horizons as keys.,The share of vehicles that uses internal combustion engines (ICE) in a given year. What is not EV or FCEV is oil-fuelled ICE.
+transport_fuel_cell _efficiency,--,float,The H2 conversion efficiencies of fuel cells in transport
+transport_internal _combustion_efficiency,--,float,The oil conversion efficiencies of internal combustion engine (ICE) in transport
+agriculture_machinery _electric_share,--,float,The share for agricultural machinery that uses electricity
+agriculture_machinery _oil_share,--,float,The share for agricultural machinery that uses oil
+agriculture_machinery _fuel_efficiency,--,float,The efficiency of electric-powered machinery in the conversion of electricity to meet agricultural needs.
+agriculture_machinery _electric_efficiency,--,float,The efficiency of oil-powered machinery in the conversion of oil to meet agricultural needs.
+Mwh_MeOH_per_MWh_H2,LHV,float,"The energy amount of the produced methanol per energy amount of hydrogen. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, page 64."
 MWh_MeOH_per_tCO2,LHV,float,"The energy amount of the produced methanol per ton of CO2. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, page 64."
 MWh_MeOH_per_MWh_e,LHV,float,"The energy amount of the produced methanol per energy amount of electricity. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, page 64."
-shipping_hydrogen_liquefaction,--,"{true, false}",Whether to include liquefaction costs for hydrogen demand in shipping.
-
+shipping_hydrogen _liquefaction,--,"{true, false}",Whether to include liquefaction costs for hydrogen demand in shipping.
+,,,
 shipping_hydrogen_share,--,Dictionary with planning horizons as keys.,The share of ships powered by hydrogen in a given year
 shipping_methanol_share,--,Dictionary with planning horizons as keys.,The share of ships powered by methanol in a given year
 shipping_oil_share,--,Dictionary with planning horizons as keys.,The share of ships powered by oil in a given year
-shipping_methanol_efficiency,--,float,"The efficiency of methanol-powered ships in the conversion of methanol to meet shipping needs (propulsion). The efficiency increase from oil can be 10-15% higher according to the `IEA <https://www.iea-amf.org/app/webroot/files/file/Annex%20Reports/AMF_Annex_56.pdf>`_"
-
+shipping_methanol _efficiency,--,float,The efficiency of methanol-powered ships in the conversion of methanol to meet shipping needs (propulsion). The efficiency increase from oil can be 10-15% higher according to the `IEA <https://www.iea-amf.org/app/webroot/files/file/Annex%20Reports/AMF_Annex_56.pdf>`_
+,,,
 shipping_oil_efficiency,--,float,The efficiency of oil-powered ships in the conversion of oil to meet shipping needs (propulsion). Base value derived from 2011
 aviation_demand_factor,--,float,The proportion of demand for aviation compared to today's consumption
 HVC_demand_factor,--,float,The proportion of demand for high-value chemicals compared to today's consumption
-
+,,,
 time_dep_hp_cop,--,"{true, false}",Consider the time dependent coefficient of performance (COP) of the heat pump
 heat_pump_sink_T,°C,float,The temperature heat sink used in heat pumps based on DTU / large area radiators. The value is conservatively high to cover hot water and space heating in poorly-insulated buildings
-reduce_space_heat_exogenously,--,"{true, false}",Influence on space heating demand by a certain factor (applied before losses in district heating).
-reduce_space_heat_exogenously_factor,--,Dictionary with planning horizons as keys.,"A positive factor can mean renovation or demolition of a building. If the factor is negative, it can mean an increase in floor area, increased thermal comfort, population growth. The default factors are determined by the `Eurocalc Homes and buildings decarbonization scenario <http://tool.european-calculator.eu/app/buildings/building-types-area/?levers=1ddd4444421213bdbbbddd44444ffffff11f411111221111211l212221>`_"
+reduce_space_heat _exogenously,--,"{true, false}",Influence on space heating demand by a certain factor (applied before losses in district heating).
+reduce_space_heat _exogenously_factor,--,Dictionary with planning horizons as keys.,"A positive factor can mean renovation or demolition of a building. If the factor is negative, it can mean an increase in floor area, increased thermal comfort, population growth. The default factors are determined by the `Eurocalc Homes and buildings decarbonization scenario <http://tool.european-calculator.eu/app/buildings/building-types-area/?levers=1ddd4444421213bdbbbddd44444ffffff11f411111221111211l212221>`_"
 retrofitting,,,
 -- retro_endogen,--,"{true, false}",Add retrofitting as an endogenous system which co-optimise space heat savings.
 -- cost_factor,--,float,Weight costs for building renovation
@@ -79,7 +79,7 @@ allam_cycle,--,"{true, false}",Add option to include `Allam cycle gas power plan
 hydrogen_fuel_cell,--,"{true, false}",Add option to include hydrogen fuel cell for re-electrification. Assuming OCGT technology costs
 hydrogen_turbine,--,"{true, false}",Add option to include hydrogen turbine for re-electrification. Assuming OCGT technology costs
 SMR,--,"{true, false}",Add option for transforming natural gas into hydrogen and CO2 using Steam Methane Reforming (SMR)
-regional_co2_sequestration_potential,,,
+regional_co2 _sequestration_potential,,,
 -- enable,--,"{true, false}",Add option for regionally-resolved geological carbon dioxide sequestration potentials based on `CO2StoP <https://setis.ec.europa.eu/european-co2-storage-database_en>`_.
 -- attribute,--,string,Name of the attribute for the sequestration potential
 -- include_onshore,--,"{true, false}",Add options for including onshore sequestration potentials
@@ -89,32 +89,32 @@ regional_co2_sequestration_potential,,,
 co2_sequestration_potential,MtCO2/a,float,The potential of sequestering CO2 in Europe per year
 co2_sequestration_cost,currency/tCO2,float,The cost of sequestering a ton of CO2
 co2_spatial,--,"{true, false}","Add option to spatially resolve carrier representing stored carbon dioxide. This allows for more detailed modelling of CCUTS, e.g. regarding the capturing of industrial process emissions, usage as feedstock for electrofuels, transport of carbon dioxide, and geological sequestration sites."
-
+,,,
 co2network,--,"{true, false}",Add option for planning a new carbon dioxide transmission network
-
+,,,
 cc_fraction,--,float,The default fraction of CO2 captured with post-combustion capture
-hydrogen_underground_storage,--,"{true, false}",Add options for storing hydrogen underground. Storage potential depends regionally.
-hydrogen_underground_storage_locations,,"{onshore, nearshore, offshore}","The location where hydrogen underground storage can be located. Onshore, nearshore, offshore means it must be located more than 50 km away from the sea, within 50 km of the sea, or within the sea itself respectively."
-
+hydrogen_underground _storage,--,"{true, false}",Add options for storing hydrogen underground. Storage potential depends regionally.
+hydrogen_underground _storage_locations,,"{onshore, nearshore, offshore}","The location where hydrogen underground storage can be located. Onshore, nearshore, offshore means it must be located more than 50 km away from the sea, within 50 km of the sea, or within the sea itself respectively."
+,,,
 ammonia,--,"{true, false, regional}","Add ammonia as a carrrier. It can be either true (copperplated NH3), false (no NH3 carrier) or ""regional"" (regionalised NH3 without network)"
-min_part_load_fischer_tropsch,per unit of p_nom ,float,The minimum unit dispatch (``p_min_pu``) for the Fischer-Tropsch process
-min_part_load_methanolisation,per unit of p_nom ,float,The minimum unit dispatch (``p_min_pu``) for the methanolisation process
-
-use_fischer_tropsch_waste_heat,--,"{true, false}",Add option for using waste heat of Fischer Tropsch in district heating networks
+min_part_load_fischer _tropsch,per unit of p_nom ,float,The minimum unit dispatch (``p_min_pu``) for the Fischer-Tropsch process
+min_part_load _methanolisation,per unit of p_nom ,float,The minimum unit dispatch (``p_min_pu``) for the methanolisation process
+,,,
+use_fischer_tropsch _waste_heat,--,"{true, false}",Add option for using waste heat of Fischer Tropsch in district heating networks
 use_fuel_cell_waste_heat,--,"{true, false}",Add option for using waste heat of fuel cells in district heating networks
-use_electrolysis_waste_heat,--,"{true, false}",Add option for using waste heat of electrolysis in district heating networks
-electricity_distribution_grid,--,"{true, false}",Add a simplified representation of the exchange capacity between transmission and distribution grid level through a link.
-electricity_distribution_grid_cost_factor,,,Multiplies the investment cost of the electricity distribution grid
-
-electricity_grid_connection,--,"{true, false}",Add the cost of electricity grid connection for onshore wind and solar
+use_electrolysis_waste _heat,--,"{true, false}",Add option for using waste heat of electrolysis in district heating networks
+electricity_distribution _grid,--,"{true, false}",Add a simplified representation of the exchange capacity between transmission and distribution grid level through a link.
+electricity_distribution _grid_cost_factor,,,Multiplies the investment cost of the electricity distribution grid
+,,,
+electricity_grid _connection,--,"{true, false}",Add the cost of electricity grid connection for onshore wind and solar
 H2_network,--,"{true, false}",Add option for new hydrogen pipelines
 gas_network,--,"{true, false}","Add existing natural gas infrastructure, incl. LNG terminals, production and entry-points. The existing gas network is added with a lossless transport model. A length-weighted `k-edge augmentation algorithm   <https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation>`_   can be run to add new candidate gas pipelines such that all regions of the model can be connected to the gas network. When activated, all the gas demands are regionally disaggregated as well."
-H2_retrofit,--,"{true, false}","Add option for retrofiting existing pipelines to transport hydrogen. "
-H2_retrofit_capacity_per_CH4,--,float,"The ratio for H2 capacity per original CH4 capacity of retrofitted pipelines. The `European Hydrogen Backbone (April, 2020) p.15 <https://gasforclimate2050.eu/wp-content/uploads/2020/07/2020_European-Hydrogen-Backbone_Report.pdf>`_ 60% of original natural gas capacity could be used in cost-optimal case as H2 capacity."
-gas_network_connectivity_upgrade ,--,float,The number of desired edge connectivity (k) in the length-weighted `k-edge augmentation algorithm <https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation>`_ used for the gas network
+H2_retrofit,--,"{true, false}",Add option for retrofiting existing pipelines to transport hydrogen. 
+H2_retrofit_capacity _per_CH4,--,float,"The ratio for H2 capacity per original CH4 capacity of retrofitted pipelines. The `European Hydrogen Backbone (April, 2020) p.15 <https://gasforclimate2050.eu/wp-content/uploads/2020/07/2020_European-Hydrogen-Backbone_Report.pdf>`_ 60% of original natural gas capacity could be used in cost-optimal case as H2 capacity."
+gas_network_connectivity _upgrade ,--,float,The number of desired edge connectivity (k) in the length-weighted `k-edge augmentation algorithm <https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation>`_ used for the gas network
 gas_distribution_grid,--,"{true, false}",Add a gas distribution grid
-gas_distribution_grid_cost_factor,,,Multiplier for the investment cost of the gas distribution grid
-
+gas_distribution_grid _cost_factor,,,Multiplier for the investment cost of the gas distribution grid
+,,,
 biomass_spatial,--,"{true, false}",Add option for resolving biomass demand regionally
 biomass_transport,--,"{true, false}",Add option for transporting solid biomass between nodes
 conventional_generation,,,Add a more detailed description of conventional carriers. Any power generation requires the consumption of fuel from nodes representing that fuel.
diff --git a/doc/configuration.rst b/doc/configuration.rst
index 0b2669f6..8f15faa7 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -24,7 +24,7 @@ Top-level configuration
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/toplevel.csv
 
 .. _run_cf:
@@ -45,7 +45,7 @@ The ``run`` section is used for running and storing scenarios with different con
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/run.csv
 
 .. _foresight_cf:
@@ -60,7 +60,7 @@ The ``run`` section is used for running and storing scenarios with different con
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/foresight.csv
 
 .. note::
@@ -105,7 +105,7 @@ An exemplary dependency graph (starting from the simplification rules) then look
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/scenario.csv
 
 .. _countries:
@@ -120,7 +120,7 @@ An exemplary dependency graph (starting from the simplification rules) then look
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/countries.csv
 
 .. _snapshots_cf:
@@ -137,7 +137,7 @@ Specifies the temporal range to build an energy system model for as arguments to
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/snapshots.csv
 
 .. _enable_cf:
@@ -154,7 +154,7 @@ Switches for some rules and optional features.
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/enable.csv
 
 .. _CO2_budget_cf:
@@ -169,7 +169,7 @@ Switches for some rules and optional features.
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/co2_budget.csv
 
 .. note::
@@ -188,7 +188,7 @@ Switches for some rules and optional features.
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/electricity.csv
 
 .. _atlite_cf:
@@ -205,7 +205,7 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/atlite.csv
 
 .. _renewable_cf:
@@ -223,7 +223,7 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/onwind.csv
 
 .. note::
@@ -245,7 +245,7 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/offwind-ac.csv
 
 .. note::
@@ -268,7 +268,7 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/offwind-dc.csv
 
 .. note::
@@ -285,7 +285,7 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/solar.csv
 
 .. note::
@@ -307,7 +307,7 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/hydro.csv
 
 .. _lines_cf:
@@ -329,7 +329,7 @@ overwrite the existing values.
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/conventional.csv
 
 ``lines``
@@ -342,7 +342,7 @@ overwrite the existing values.
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/lines.csv
 
 .. _links_cf:
@@ -357,7 +357,7 @@ overwrite the existing values.
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/links.csv
 
 .. _transformers_cf:
@@ -372,7 +372,7 @@ overwrite the existing values.
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/transformers.csv
 
 .. _load_cf:
@@ -387,7 +387,7 @@ overwrite the existing values.
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/load.csv
 
 .. _energy_cf:
@@ -405,7 +405,7 @@ overwrite the existing values.
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/energy.csv
 
 .. _biomass_cf:
@@ -423,7 +423,7 @@ overwrite the existing values.
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/biomass.csv
 
 The list of available biomass is given by the category in `ENSPRESO_BIOMASS <https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx>`_, namely:
@@ -461,7 +461,7 @@ The list of available biomass is given by the category in `ENSPRESO_BIOMASS <htt
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/solar-thermal.csv
 
 .. _existing_capacities_cf:
@@ -479,7 +479,7 @@ The list of available biomass is given by the category in `ENSPRESO_BIOMASS <htt
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/existing_capacities.csv
 
 .. _sector_cf:
@@ -497,7 +497,7 @@ The list of available biomass is given by the category in `ENSPRESO_BIOMASS <htt
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/sector.csv
 
 .. _industry_cf:
@@ -515,7 +515,7 @@ The list of available biomass is given by the category in `ENSPRESO_BIOMASS <htt
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/industry.csv
 
 .. _costs_cf:
@@ -530,7 +530,7 @@ The list of available biomass is given by the category in `ENSPRESO_BIOMASS <htt
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/costs.csv
 
 .. note::
@@ -549,7 +549,7 @@ The list of available biomass is given by the category in `ENSPRESO_BIOMASS <htt
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/clustering.csv
 
 .. note::
@@ -572,7 +572,7 @@ The list of available biomass is given by the category in `ENSPRESO_BIOMASS <htt
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/solving.csv
 
 .. _plotting_cf:
@@ -589,5 +589,5 @@ The list of available biomass is given by the category in `ENSPRESO_BIOMASS <htt
 
 .. csv-table::
    :header-rows: 1
-   :widths: 25,7,22,30
+   :widths: 22,7,22,33
    :file: configtables/plotting.csv
diff --git a/doc/tutorial.rst b/doc/tutorial.rst
index 518c58a3..f0ded3fb 100644
--- a/doc/tutorial.rst
+++ b/doc/tutorial.rst
@@ -32,7 +32,7 @@ configuration, execute
 .. code:: bash
     :class: full-width
 
-    snakemake -call results/test-elec/networks/elec_s_6_ec_lcopt_Co2L-24H.nc --configfile test/config.electricity.yaml
+    snakemake -call results/test-elec/networks/elec_s_6_ec_lcopt_Co2L-24H.nc --configfile config/test/config.electricity.yaml
 
 This configuration is set to download a reduced data set via the rules :mod:`retrieve_databundle`,
 :mod:`retrieve_natura_raster`, :mod:`retrieve_cutout`.
@@ -115,7 +115,7 @@ clustered down to 6 buses and every 24 hours aggregated to one snapshot. The com
 
 .. code:: bash
 
-    snakemake -call results/test-elec/networks/elec_s_6_ec_lcopt_Co2L-24H.nc --configfile test/config.electricity.yaml
+    snakemake -call results/test-elec/networks/elec_s_6_ec_lcopt_Co2L-24H.nc --configfile config/test/config.electricity.yaml
 
 orders ``snakemake`` to run the rule :mod:`solve_network` that produces the solved network and stores it in ``results/networks`` with the name ``elec_s_6_ec_lcopt_Co2L-24H.nc``:
 
@@ -276,18 +276,18 @@ You can produce any output file occurring in the ``Snakefile`` by running
 
 For example, you can explore the evolution of the PyPSA networks by running
 
-#. ``snakemake resources/networks/base.nc -call --configfile test/config.electricity.yaml``
-#. ``snakemake resources/networks/elec.nc -call --configfile test/config.electricity.yaml``
-#. ``snakemake resources/networks/elec_s.nc -call --configfile test/config.electricity.yaml``
-#. ``snakemake resources/networks/elec_s_6.nc -call --configfile test/config.electricity.yaml``
-#. ``snakemake resources/networks/elec_s_6_ec_lcopt_Co2L-24H.nc -call --configfile test/config.electricity.yaml``
+#. ``snakemake resources/networks/base.nc -call --configfile config/test/config.electricity.yaml``
+#. ``snakemake resources/networks/elec.nc -call --configfile config/test/config.electricity.yaml``
+#. ``snakemake resources/networks/elec_s.nc -call --configfile config/test/config.electricity.yaml``
+#. ``snakemake resources/networks/elec_s_6.nc -call --configfile config/test/config.electricity.yaml``
+#. ``snakemake resources/networks/elec_s_6_ec_lcopt_Co2L-24H.nc -call --configfile config/test/config.electricity.yaml``
 
 To run all combinations of wildcard values provided in the ``config/config.yaml`` under ``scenario:``,
 you can use the collection rule ``solve_elec_networks``.
 
 .. code:: bash
 
-    snakemake -call solve_elec_networks --configfile test/config.electricity.yaml
+    snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml
 
 If you now feel confident and want to tackle runs with larger temporal and
 spatial scope, clean-up the repository and after modifying the ``config/config.yaml`` file
diff --git a/doc/tutorial_sector.rst b/doc/tutorial_sector.rst
index 29971e3a..faa8adca 100644
--- a/doc/tutorial_sector.rst
+++ b/doc/tutorial_sector.rst
@@ -35,7 +35,7 @@ configuration options. In the example below, we say that the gas network should
 be added and spatially resolved. We also say that the existing gas network may
 be retrofitted to transport hydrogen instead.
 
-.. literalinclude:: ../test/config.overnight.yaml
+.. literalinclude:: ../config/test/config.overnight.yaml
    :language: yaml
    :start-at: sector:
    :end-before: solving:
@@ -45,7 +45,7 @@ Documentation for all options will be added successively to :ref:`config`.
 Scenarios can be defined like for electricity-only studies, but with additional
 wildcard options.
 
-.. literalinclude:: ../test/config.overnight.yaml
+.. literalinclude:: ../config/test/config.overnight.yaml
    :language: yaml
    :start-at: scenario:
    :end-before: countries:
@@ -59,7 +59,7 @@ To run an overnight / greenfiled scenario with the specifications above, run
 
 .. code:: bash
 
-    snakemake -call --configfile test/config.overnight.yaml all
+    snakemake -call --configfile config/test/config.overnight.yaml all
 
 which will result in the following *additional* jobs ``snakemake`` wants to run
 on top of those already included in the electricity-only tutorial:
@@ -294,7 +294,7 @@ Scenarios can be defined like for electricity-only studies, but with additional
 wildcard options. For the myopic foresight mode, the ``{planning_horizons}`` wildcard
 defines the sequence of investment horizons.
 
-.. literalinclude:: ../test/config.myopic.yaml
+.. literalinclude:: ../config/test/config.myopic.yaml
    :language: yaml
    :start-at: scenario:
    :end-before: countries:
@@ -304,7 +304,7 @@ For allowed wildcard values, refer to :ref:`wildcards`.
 In the myopic foresight mode, you can tweak for instance exogenously given transition paths, like the one for
 the share of primary steel production we change below:
 
-.. literalinclude:: ../test/config.myopic.yaml
+.. literalinclude:: ../config/test/config.myopic.yaml
    :language: yaml
    :start-at: industry:
    :end-before: solving:
@@ -318,7 +318,7 @@ To run a myopic foresight scenario with the specifications above, run
 
 .. code:: bash
 
-    snakemake -call --configfile test/config.myopic.yaml all
+    snakemake -call --configfile config/test/config.myopic.yaml all
 
 which will result in the following *additional* jobs ``snakemake`` wants to run:
 

From 5482c27e8b4d7df34c0bc3bd00be0fc385999843 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 14 Jul 2023 10:26:07 +0000
Subject: [PATCH 168/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 doc/configtables/sector.csv | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/configtables/sector.csv b/doc/configtables/sector.csv
index 36b7dd98..d610c862 100644
--- a/doc/configtables/sector.csv
+++ b/doc/configtables/sector.csv
@@ -109,7 +109,7 @@ electricity_distribution _grid_cost_factor,,,Multiplies the investment cost of t
 electricity_grid _connection,--,"{true, false}",Add the cost of electricity grid connection for onshore wind and solar
 H2_network,--,"{true, false}",Add option for new hydrogen pipelines
 gas_network,--,"{true, false}","Add existing natural gas infrastructure, incl. LNG terminals, production and entry-points. The existing gas network is added with a lossless transport model. A length-weighted `k-edge augmentation algorithm   <https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation>`_   can be run to add new candidate gas pipelines such that all regions of the model can be connected to the gas network. When activated, all the gas demands are regionally disaggregated as well."
-H2_retrofit,--,"{true, false}",Add option for retrofiting existing pipelines to transport hydrogen. 
+H2_retrofit,--,"{true, false}",Add option for retrofiting existing pipelines to transport hydrogen.
 H2_retrofit_capacity _per_CH4,--,float,"The ratio for H2 capacity per original CH4 capacity of retrofitted pipelines. The `European Hydrogen Backbone (April, 2020) p.15 <https://gasforclimate2050.eu/wp-content/uploads/2020/07/2020_European-Hydrogen-Backbone_Report.pdf>`_ 60% of original natural gas capacity could be used in cost-optimal case as H2 capacity."
 gas_network_connectivity _upgrade ,--,float,The number of desired edge connectivity (k) in the length-weighted `k-edge augmentation algorithm <https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation>`_ used for the gas network
 gas_distribution_grid,--,"{true, false}",Add a gas distribution grid

From 328e413b12d7dcf107e9395c1d64ff4362da6ff5 Mon Sep 17 00:00:00 2001
From: euronion <42553970+euronion@users.noreply.github.com>
Date: Fri, 14 Jul 2023 13:41:22 +0200
Subject: [PATCH 169/176] Update release_notes.rst

---
 doc/release_notes.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index b7fcf2fa..2e114b1a 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -40,6 +40,10 @@ Upcoming Release
   e.g. by setting ``solving: options: transmission_losses: 2`` for an
   approximation with two tangents.
 
+* Added configuration option ``enable: retrieve:`` to control whether data
+  retrieval rules from snakemake are enabled or not. Th default setting ``auto`` 
+  will automatically detect and enable/disable the rules based on internet connectivity.
+
 
 PyPSA-Eur 0.8.0 (18th March 2023)
 =================================

From 7437f32c0fd3c9d0aec994c2891b02f8f4ec6ec2 Mon Sep 17 00:00:00 2001
From: euronion <42553970+euronion@users.noreply.github.com>
Date: Fri, 14 Jul 2023 13:41:25 +0200
Subject: [PATCH 170/176] Update enable.csv

---
 doc/configtables/enable.csv | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/configtables/enable.csv b/doc/configtables/enable.csv
index 8a543b46..e1349fef 100644
--- a/doc/configtables/enable.csv
+++ b/doc/configtables/enable.csv
@@ -1,4 +1,5 @@
 ,Unit,Values,Description
+enable,str or bool,"{auto, true, false}","Switch to include (true) or exclude (false) the retrieve_* rules of snakemake into the workflow; 'auto' sets true|false based on availability of an internet connection to prevent issues with snakemake failing due to lack of internet connection."
 prepare_links_p_nom,bool,"{true, false}","Switch to retrieve current HVDC projects from `Wikipedia <https://en.wikipedia.org/wiki/List_of_HVDC_projects>`_"
 retrieve_databundle,bool,"{true, false}","Switch to retrieve databundle from zenodo via the rule :mod:`retrieve_databundle` or whether to keep a custom databundle located in the corresponding folder."
 retrieve_sector_databundle,bool,"{true, false}","Switch to retrieve sector databundle from zenodo via the rule :mod:`retrieve_sector_databundle` or whether to keep a custom databundle located in the corresponding folder."

From aa7a2c823acec4d805f08e92e34e48f740cd32a5 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 14 Jul 2023 11:42:06 +0000
Subject: [PATCH 171/176] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 doc/release_notes.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index 2e114b1a..4c7219e7 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -41,7 +41,7 @@ Upcoming Release
   approximation with two tangents.
 
 * Added configuration option ``enable: retrieve:`` to control whether data
-  retrieval rules from snakemake are enabled or not. Th default setting ``auto`` 
+  retrieval rules from snakemake are enabled or not. Th default setting ``auto``
   will automatically detect and enable/disable the rules based on internet connectivity.
 
 

From f566142d386c14994127f28cd26e706ddd133a73 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Fri, 14 Jul 2023 15:47:41 +0200
Subject: [PATCH 172/176] cluster network: update to new clustering module
 (pypsa v0.25)

---
 scripts/_helpers.py         | 17 --------------
 scripts/cluster_network.py  | 37 ++++++++---------------------
 scripts/simplify_network.py | 47 +++++++++++++++++--------------------
 3 files changed, 32 insertions(+), 69 deletions(-)

diff --git a/scripts/_helpers.py b/scripts/_helpers.py
index 0e1d2327..cd702950 100644
--- a/scripts/_helpers.py
+++ b/scripts/_helpers.py
@@ -277,23 +277,6 @@ def progress_retrieve(url, file, disable=False):
             urllib.request.urlretrieve(url, file, reporthook=update_to)
 
 
-def get_aggregation_strategies(aggregation_strategies):
-    # default aggregation strategies that cannot be defined in .yaml format must be specified within
-    # the function, otherwise (when defaults are passed in the function's definition) they get lost
-    # when custom values are specified in the config.
-
-    import numpy as np
-    from pypsa.clustering.spatial import _make_consense
-
-    bus_strategies = dict(country=_make_consense("Bus", "country"))
-    bus_strategies.update(aggregation_strategies.get("buses", {}))
-
-    generator_strategies = {"build_year": lambda x: 0, "lifetime": lambda x: np.inf}
-    generator_strategies.update(aggregation_strategies.get("generators", {}))
-
-    return bus_strategies, generator_strategies
-
-
 def mock_snakemake(rulename, configfiles=[], **wildcards):
     """
     This function is expected to be executed from the 'scripts'-directory of '
diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py
index 14741f9e..d82f7569 100644
--- a/scripts/cluster_network.py
+++ b/scripts/cluster_network.py
@@ -133,12 +133,13 @@ import pandas as pd
 import pyomo.environ as po
 import pypsa
 import seaborn as sns
-from _helpers import configure_logging, get_aggregation_strategies, update_p_nom_max
+from _helpers import configure_logging, update_p_nom_max
 from pypsa.clustering.spatial import (
     busmap_by_greedy_modularity,
     busmap_by_hac,
     busmap_by_kmeans,
     get_clustering_from_busmap,
+    make_consense,
 )
 
 warnings.filterwarnings(action="ignore", category=UserWarning)
@@ -395,10 +396,6 @@ def clustering_for_n_clusters(
     extended_link_costs=0,
     focus_weights=None,
 ):
-    bus_strategies, generator_strategies = get_aggregation_strategies(
-        aggregation_strategies
-    )
-
     if not isinstance(custom_busmap, pd.Series):
         busmap = busmap_for_n_clusters(
             n, n_clusters, solver_name, focus_weights, algorithm, feature
@@ -409,12 +406,12 @@ def clustering_for_n_clusters(
     clustering = get_clustering_from_busmap(
         n,
         busmap,
-        bus_strategies=bus_strategies,
+        bus_strategies={"country": make_consense},
         aggregate_generators_weighted=True,
         aggregate_generators_carriers=aggregate_carriers,
         aggregate_one_ports=["Load", "StorageUnit"],
         line_length_factor=line_length_factor,
-        generator_strategies=generator_strategies,
+        generator_strategies=aggregation_strategies["generators"],
         scale_link_capital_costs=False,
     )
 
@@ -460,7 +457,7 @@ if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
 
-        snakemake = mock_snakemake("cluster_network", simpl="", clusters="5")
+        snakemake = mock_snakemake("cluster_network", simpl="", clusters="37c")
     configure_logging(snakemake)
 
     params = snakemake.params
@@ -470,11 +467,13 @@ if __name__ == "__main__":
 
     exclude_carriers = params.cluster_network["exclude_carriers"]
     aggregate_carriers = set(n.generators.carrier) - set(exclude_carriers)
+    conventional_carriers = set(params.conventional_carriers)
     if snakemake.wildcards.clusters.endswith("m"):
         n_clusters = int(snakemake.wildcards.clusters[:-1])
-        aggregate_carriers = set(params.conventional_carriers).intersection(
-            aggregate_carriers
-        )
+        aggregate_carriers = params.conventional_carriers & aggregate_carriers
+    elif snakemake.wildcards.clusters.endswith("c"):
+        n_clusters = int(snakemake.wildcards.clusters[:-1])
+        aggregate_carriers = aggregate_carriers - conventional_carriers
     elif snakemake.wildcards.clusters == "all":
         n_clusters = len(n.buses)
     else:
@@ -497,22 +496,6 @@ if __name__ == "__main__":
             Nyears,
         ).at["HVAC overhead", "capital_cost"]
 
-        def consense(x):
-            v = x.iat[0]
-            assert (
-                x == v
-            ).all() or x.isnull().all(), "The `potential` configuration option must agree for all renewable carriers, for now!"
-            return v
-
-        # translate str entries of aggregation_strategies to pd.Series functions:
-        aggregation_strategies = {
-            p: {
-                k: getattr(pd.Series, v)
-                for k, v in params.aggregation_strategies[p].items()
-            }
-            for p in params.aggregation_strategies.keys()
-        }
-
         custom_busmap = params.custom_busmap
         if custom_busmap:
             custom_busmap = pd.read_csv(
diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index b4c0829b..45e84dd8 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -92,14 +92,14 @@ import numpy as np
 import pandas as pd
 import pypsa
 import scipy as sp
-from _helpers import configure_logging, get_aggregation_strategies, update_p_nom_max
+from _helpers import configure_logging, update_p_nom_max
 from add_electricity import load_costs
 from cluster_network import cluster_regions, clustering_for_n_clusters
 from pypsa.clustering.spatial import (
-    aggregategenerators,
     aggregateoneport,
     busmap_by_stubs,
     get_clustering_from_busmap,
+    make_consense,
 )
 from pypsa.io import import_components_from_dataframe, import_series_from_dataframe
 from scipy.sparse.csgraph import connected_components, dijkstra
@@ -253,11 +253,15 @@ def _aggregate_and_move_components(
 
     _adjust_capital_costs_using_connection_costs(n, connection_costs_to_bus, output)
 
-    _, generator_strategies = get_aggregation_strategies(aggregation_strategies)
+    generator_strategies = aggregation_strategies["generators"]
 
     carriers = set(n.generators.carrier) - set(exclude_carriers)
-    generators, generators_pnl = aggregategenerators(
-        n, busmap, carriers=carriers, custom_strategies=generator_strategies
+    generators, generators_pnl = aggregateoneport(
+        n,
+        busmap,
+        "Generator",
+        carriers=carriers,
+        custom_strategies=generator_strategies,
     )
 
     replace_components(n, "Generator", generators, generators_pnl)
@@ -478,9 +482,7 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None):
     busmap = n.buses.index.to_series()
     busmap.loc[buses_i] = dist.idxmin(1)
 
-    bus_strategies, generator_strategies = get_aggregation_strategies(
-        aggregation_strategies
-    )
+    bus_strategies = {"country": make_consense}
 
     clustering = get_clustering_from_busmap(
         n,
@@ -534,15 +536,6 @@ if __name__ == "__main__":
     n = pypsa.Network(snakemake.input.network)
     Nyears = n.snapshot_weightings.objective.sum() / 8760
 
-    # translate str entries of aggregation_strategies to pd.Series functions:
-    aggregation_strategies = {
-        p: {
-            k: getattr(pd.Series, v)
-            for k, v in params.aggregation_strategies[p].items()
-        }
-        for p in params.aggregation_strategies.keys()
-    }
-
     n, trafo_map = simplify_network_to_380(n)
 
     technology_costs = load_costs(
@@ -560,7 +553,7 @@ if __name__ == "__main__":
         params.p_max_pu,
         params.simplify_network["exclude_carriers"],
         snakemake.output,
-        aggregation_strategies,
+        params.aggregation_strategies,
     )
 
     busmaps = [trafo_map, simplify_links_map]
@@ -573,12 +566,12 @@ if __name__ == "__main__":
             params.length_factor,
             params.simplify_network,
             snakemake.output,
-            aggregation_strategies=aggregation_strategies,
+            aggregation_strategies=params.aggregation_strategies,
         )
         busmaps.append(stub_map)
 
     if params.simplify_network["to_substations"]:
-        n, substation_map = aggregate_to_substations(n, aggregation_strategies)
+        n, substation_map = aggregate_to_substations(n, params.aggregation_strategies)
         busmaps.append(substation_map)
 
     # treatment of outliers (nodes without a profile for considered carrier):
@@ -592,7 +585,9 @@ if __name__ == "__main__":
             logger.info(
                 f"clustering preparation (hac): aggregating {len(buses_i)} buses of type {carrier}."
             )
-            n, busmap_hac = aggregate_to_substations(n, aggregation_strategies, buses_i)
+            n, busmap_hac = aggregate_to_substations(
+                n, params.aggregation_strategies, buses_i
+            )
             busmaps.append(busmap_hac)
 
     if snakemake.wildcards.simpl:
@@ -603,20 +598,22 @@ if __name__ == "__main__":
             solver_name,
             params.simplify_network["algorithm"],
             params.simplify_network["feature"],
-            aggregation_strategies,
+            params.aggregation_strategies,
         )
         busmaps.append(cluster_map)
 
     # some entries in n.buses are not updated in previous functions, therefore can be wrong. as they are not needed
     # and are lost when clustering (for example with the simpl wildcard), we remove them for consistency:
-    buses_c = {
+    remove = [
         "symbol",
         "tags",
         "under_construction",
         "substation_lv",
         "substation_off",
-    }.intersection(n.buses.columns)
-    n.buses = n.buses.drop(buses_c, axis=1)
+        "geometry",
+    ]
+    n.buses.drop(remove, axis=1, inplace=True, errors="ignore")
+    n.lines.drop(remove, axis=1, errors="ignore", inplace=True)
 
     update_p_nom_max(n)
 

From 8e568cee0359dd6f348cb09b58a6ca73e07a7a14 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Fri, 14 Jul 2023 16:00:21 +0200
Subject: [PATCH 173/176] cluster network: remove bus strategies as already
 handled internally

---
 scripts/cluster_network.py  | 2 --
 scripts/simplify_network.py | 8 ++------
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py
index d82f7569..ea749cbc 100644
--- a/scripts/cluster_network.py
+++ b/scripts/cluster_network.py
@@ -139,7 +139,6 @@ from pypsa.clustering.spatial import (
     busmap_by_hac,
     busmap_by_kmeans,
     get_clustering_from_busmap,
-    make_consense,
 )
 
 warnings.filterwarnings(action="ignore", category=UserWarning)
@@ -406,7 +405,6 @@ def clustering_for_n_clusters(
     clustering = get_clustering_from_busmap(
         n,
         busmap,
-        bus_strategies={"country": make_consense},
         aggregate_generators_weighted=True,
         aggregate_generators_carriers=aggregate_carriers,
         aggregate_one_ports=["Load", "StorageUnit"],
diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 45e84dd8..9315342f 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -86,7 +86,7 @@ The rule :mod:`simplify_network` does up to four things:
 """
 
 import logging
-from functools import reduce
+from functools import partial, reduce
 
 import numpy as np
 import pandas as pd
@@ -99,7 +99,6 @@ from pypsa.clustering.spatial import (
     aggregateoneport,
     busmap_by_stubs,
     get_clustering_from_busmap,
-    make_consense,
 )
 from pypsa.io import import_components_from_dataframe, import_series_from_dataframe
 from scipy.sparse.csgraph import connected_components, dijkstra
@@ -482,17 +481,14 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None):
     busmap = n.buses.index.to_series()
     busmap.loc[buses_i] = dist.idxmin(1)
 
-    bus_strategies = {"country": make_consense}
-
     clustering = get_clustering_from_busmap(
         n,
         busmap,
-        bus_strategies=bus_strategies,
         aggregate_generators_weighted=True,
         aggregate_generators_carriers=None,
         aggregate_one_ports=["Load", "StorageUnit"],
         line_length_factor=1.0,
-        generator_strategies=generator_strategies,
+        generator_strategies=aggregation_strategies["generators"],
         scale_link_capital_costs=False,
     )
     return clustering.network, busmap

From d22c87419b6093927595a827160b06eb0109a22a Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Mon, 17 Jul 2023 14:03:16 +0200
Subject: [PATCH 174/176] update enviromnent yaml to install pypsa from master
 cluster: allow custom strategies for all components

---
 envs/environment.yaml       | 1 +
 scripts/cluster_network.py  | 8 +++++++-
 scripts/simplify_network.py | 8 +++++++-
 3 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/envs/environment.yaml b/envs/environment.yaml
index 9d800fdc..68540801 100644
--- a/envs/environment.yaml
+++ b/envs/environment.yaml
@@ -56,3 +56,4 @@ dependencies:
 
 - pip:
   - tsam>=1.1.0
+  - git+https://github.com/PyPSA/PyPSA.git@master
diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py
index ea749cbc..f0fd80eb 100644
--- a/scripts/cluster_network.py
+++ b/scripts/cluster_network.py
@@ -402,6 +402,10 @@ def clustering_for_n_clusters(
     else:
         busmap = custom_busmap
 
+    line_strategies = aggregation_strategies.get("lines", dict())
+    generator_strategies = aggregation_strategies.get("generators", dict())
+    one_port_strategies = aggregation_strategies.get("one_ports", dict())
+
     clustering = get_clustering_from_busmap(
         n,
         busmap,
@@ -409,7 +413,9 @@ def clustering_for_n_clusters(
         aggregate_generators_carriers=aggregate_carriers,
         aggregate_one_ports=["Load", "StorageUnit"],
         line_length_factor=line_length_factor,
-        generator_strategies=aggregation_strategies["generators"],
+        line_strategies=line_strategies,
+        generator_strategies=generator_strategies,
+        one_port_strategies=one_port_strategies,
         scale_link_capital_costs=False,
     )
 
diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 9315342f..9fce8e06 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -481,6 +481,10 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None):
     busmap = n.buses.index.to_series()
     busmap.loc[buses_i] = dist.idxmin(1)
 
+    line_strategies = aggregation_strategies.get("lines", dict())
+    generator_strategies = aggregation_strategies.get("generators", dict())
+    one_port_strategies = aggregation_strategies.get("one_ports", dict())
+
     clustering = get_clustering_from_busmap(
         n,
         busmap,
@@ -488,7 +492,9 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None):
         aggregate_generators_carriers=None,
         aggregate_one_ports=["Load", "StorageUnit"],
         line_length_factor=1.0,
-        generator_strategies=aggregation_strategies["generators"],
+        line_strategies=line_strategies,
+        generator_strategies=generator_strategies,
+        one_port_strategies=one_port_strategies,
         scale_link_capital_costs=False,
     )
     return clustering.network, busmap

From 00cb865ce05374a4b79ef95207ad838bd4263ff5 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Mon, 17 Jul 2023 15:37:30 +0200
Subject: [PATCH 175/176] env: force use pypsa from GH master

---
 envs/environment.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/envs/environment.yaml b/envs/environment.yaml
index 68540801..c8e469ac 100644
--- a/envs/environment.yaml
+++ b/envs/environment.yaml
@@ -10,7 +10,7 @@ dependencies:
 - python>=3.8
 - pip
 
-- pypsa>=0.23
+# - pypsa>=0.23
 - atlite>=0.2.9
 - dask
 

From e5ce76bf480670940cc6020ef76f886e325bd1ac Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 17 Jul 2023 23:29:16 +0000
Subject: [PATCH 176/176] [pre-commit.ci] pre-commit autoupdate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

updates:
- [github.com/PyCQA/docformatter: v1.7.4 → v1.7.5](https://github.com/PyCQA/docformatter/compare/v1.7.4...v1.7.5)
- [github.com/psf/black: 23.3.0 → 23.7.0](https://github.com/psf/black/compare/23.3.0...23.7.0)
- [github.com/macisamuele/language-formatters-pre-commit-hooks: v2.9.0 → v2.10.0](https://github.com/macisamuele/language-formatters-pre-commit-hooks/compare/v2.9.0...v2.10.0)
---
 .pre-commit-config.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index cf811240..6fb0dbf0 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -39,7 +39,7 @@ repos:
 
   # Make docstrings PEP 257 compliant
 - repo: https://github.com/PyCQA/docformatter
-  rev: v1.7.4
+  rev: v1.7.5
   hooks:
   - id: docformatter
     args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"]
@@ -51,7 +51,7 @@ repos:
 
   # Formatting with "black" coding style
 - repo: https://github.com/psf/black
-  rev: 23.3.0
+  rev: 23.7.0
   hooks:
       # Format Python files
   - id: black
@@ -67,7 +67,7 @@ repos:
 
   # Do YAML formatting (before the linter checks it for misses)
 - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
-  rev: v2.9.0
+  rev: v2.10.0
   hooks:
   - id: pretty-format-yaml
     args: [--autofix, --indent, "2", --preserve-quotes]