Merge branch 'master' into carbon-management

.github/workflows/ci.yaml

@@ -52,10 +52,9 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Clone pypsa-eur and technology-data repositories
+      - name: Clone pypsa-eur subworkflow
         run: |
           git clone https://github.com/pypsa/pypsa-eur ../pypsa-eur
-          git clone https://github.com/pypsa/technology-data ../technology-data
           cp ../pypsa-eur/test/config.test1.yaml ../pypsa-eur/config.yaml
       
       - name: Setup secrets

Snakefile

@@ -1,6 +1,6 @@
 
 from os.path import exists
-from shutil import copyfile
+from shutil import copyfile, move
 
 from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider
 HTTP = HTTPRemoteProvider()
@@ -21,7 +21,6 @@ wildcard_constraints:
 
 SDIR = config['summary_dir'] + '/' + config['run']
 RDIR = config['results_dir'] + config['run']
-CDIR = config['costs_dir']
 
 
 subworkflow pypsaeur:
@@ -72,6 +71,15 @@ if config.get('retrieve_sector_databundle', True):
         script: 'scripts/retrieve_sector_databundle.py'
 
 
+if config.get("retrieve_cost_data", True):
+    rule retrieve_cost_data:
+        input: HTTP.remote("raw.githubusercontent.com/PyPSA/technology-data/{}/outputs/".format(config['costs']['version']) + "costs_{year}.csv", keep_local=True)
+        output: "data/costs_{year}.csv"
+        log: "logs/" + RDIR + "retrieve_cost_data_{year}.log",
+        resources: mem_mb=1000,
+        run: move(input[0], output[0])
+
+
 rule build_population_layouts:
     input:
         nuts3_shapes=pypsaeur('resources/nuts3_shapes.geojson'),
@@ -500,7 +508,7 @@ rule prepare_sector_network:
         co2="data/eea/UNFCCC_v23.csv",
         biomass_potentials='resources/biomass_potentials_s{simpl}_{clusters}.csv',
         heat_profile="data/heat_load_profile_BDEW.csv",
-        costs=CDIR + "costs_{}.csv".format(config['costs']['year']) if config["foresight"] == "overnight" else CDIR + "costs_{planning_horizons}.csv",
+        costs="data/costs_{}.csv".format(config['costs']['year']) if config["foresight"] == "overnight" else "data/costs_{planning_horizons}.csv",
         profile_offwind_ac=pypsaeur("resources/profile_offwind-ac.nc"),
         profile_offwind_dc=pypsaeur("resources/profile_offwind-dc.nc"),
         h2_cavern="resources/salt_cavern_potentials_s{simpl}_{clusters}.csv",
@@ -575,7 +583,7 @@ rule make_summary:
             RDIR + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc",
             **config['scenario']
         ),
-        costs=CDIR + "costs_{}.csv".format(config['costs']['year']) if config["foresight"] == "overnight" else CDIR + "costs_{}.csv".format(config['scenario']['planning_horizons'][0]),
+        costs="data/costs_{}.csv".format(config['costs']['year']) if config["foresight"] == "overnight" else "data/costs_{}.csv".format(config['scenario']['planning_horizons'][0]),
         plots=expand(
             RDIR + "/maps/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf",
             **config['scenario']
@@ -625,7 +633,7 @@ if config["foresight"] == "overnight":
         input:
             overrides="data/override_component_attrs",
             network=RDIR + "/prenetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc",
-            costs=CDIR + "costs_{}.csv".format(config['costs']['year']),
+            costs="data/costs_{}.csv".format(config['costs']['year']),
             config=SDIR + '/configs/config.yaml',
             #env=SDIR + '/configs/environment.yaml',
         output: RDIR + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc"
@@ -650,7 +658,7 @@ if config["foresight"] == "myopic":
             busmap_s=pypsaeur("resources/busmap_elec_s{simpl}.csv"),
             busmap=pypsaeur("resources/busmap_elec_s{simpl}_{clusters}.csv"),
             clustered_pop_layout="resources/pop_layout_elec_s{simpl}_{clusters}.csv",
-            costs=CDIR + "costs_{}.csv".format(config['scenario']['planning_horizons'][0]),
+            costs="data/costs_{}.csv".format(config['scenario']['planning_horizons'][0]),
             cop_soil_total="resources/cop_soil_total_elec_s{simpl}_{clusters}.nc",
             cop_air_total="resources/cop_air_total_elec_s{simpl}_{clusters}.nc",
             existing_heating='data/existing_infrastructure/existing_heating_raw.csv',
@@ -679,7 +687,7 @@ if config["foresight"] == "myopic":
             overrides="data/override_component_attrs",
             network=RDIR + '/prenetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc',
             network_p=solved_previous_horizon, #solved network at previous time step
-            costs=CDIR + "costs_{planning_horizons}.csv",
+            costs="data/costs_{planning_horizons}.csv",
             cop_soil_total="resources/cop_soil_total_elec_s{simpl}_{clusters}.nc",
             cop_air_total="resources/cop_air_total_elec_s{simpl}_{clusters}.nc"
         output: RDIR + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc"
@@ -696,7 +704,7 @@ if config["foresight"] == "myopic":
         input:
             overrides="data/override_component_attrs",
             network=RDIR + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc",
-            costs=CDIR + "costs_{planning_horizons}.csv",
+            costs="data/costs_{planning_horizons}.csv",
             config=SDIR + '/configs/config.yaml'
         output: RDIR + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc"
         shadow: "shallow"

config.default.yaml

@@ -3,10 +3,10 @@ version: 0.6.0
 logging_level: INFO
 
 retrieve_sector_databundle: true
+retrieve_cost_data: true
 
 results_dir: results/
 summary_dir: results
-costs_dir: ../technology-data/outputs/
 run: your-run-name  # use this to keep track of runs with different settings
 foresight: overnight # options are overnight, myopic, perfect (perfect is not yet implemented)
 # if you use myopic or perfect foresight, set the investment years in "planning_horizons" below
@@ -66,7 +66,7 @@ snapshots:
   # arguments to pd.date_range
   start: "2013-01-01"
   end: "2014-01-01"
-  closed: left # end is not inclusive
+  inclusive: left # end is not inclusive
 
 atlite:
   cutout: ../pypsa-eur/cutouts/europe-2013-era5.nc
@@ -160,6 +160,7 @@ sector:
       2040: 0.6
       2050: 1.0
     district_heating_loss: 0.15
+  cluster_heat_buses: false # cluster residential and service heat buses to one to save memory
   bev_dsm_restriction_value: 0.75  #Set to 0 for no restriction on BEV DSM
   bev_dsm_restriction_time: 7  #Time at which SOC of BEV has to be dsm_restriction_value
   transport_heating_deadband_upper: 20.
@@ -262,8 +263,11 @@ sector:
     - nearshore  # within 50 km of sea
     # - offshore
   ammonia: false # can be false (no NH3 carrier), true (copperplated NH3), "regional" (regionalised NH3 without network)
+  min_part_load_fischer_tropsch: 0.9 # p_min_pu
+  min_part_load_methanolisation: 0.5 # p_min_pu
   use_fischer_tropsch_waste_heat: true
   use_fuel_cell_waste_heat: true
+  use_electrolysis_waste_heat: false
   electricity_distribution_grid: true
   electricity_distribution_grid_cost_factor: 1.0  #multiplies cost in data/costs.csv
   electricity_grid_connection: true  # only applies to onshore wind and utility PV
@@ -341,6 +345,7 @@ industry:
 
 costs:
   year: 2030
+  version: v0.5.0
   lifetime: 25 #default lifetime
   # From a Lion Hirth paper, also reflects average of Noothout et al 2016
   discountrate: 0.07

doc/installation.rst

@@ -25,15 +25,6 @@ then download and unpack all the PyPSA-Eur data files by running the following s
     projects/pypsa-eur % snakemake -j 1 retrieve_databundle
 
 
-Clone technology-data repository
-================================
-
-Next install the technology assumptions database `technology-data <https://github.com/PyPSA/technology-data>`_ by creating a parallel directory:
-
-.. code:: bash
-
-    projects % git clone https://github.com/PyPSA/technology-data.git
-
 
 Clone PyPSA-Eur-Sec repository
 ==============================

doc/myopic.rst

@@ -81,12 +81,12 @@ Conventional carriers indicate carriers used in the existing conventional techno
 Options
 =============
 The total carbon budget for the entire transition path can be indicated in the `sector_opts <https://github.com/PyPSA/pypsa-eur-sec/blob/f13902510010b734c510c38c4cae99356f683058/config.default.yaml#L25>`_ in ``config.yaml``. The carbon budget can be split among the ``planning_horizons`` following an exponential or beta decay. 
-E.g. ``'cb40ex0'`` splits a carbon budget equal to 40 GtCO_2 following an exponential decay whose initial linear growth rate $r$ is zero.
+E.g. ``'cb40ex0'`` splits a carbon budget equal to 40 Gt :math:`_{CO_2}` following an exponential decay whose initial linear growth rate r is zero.
 They can also follow some user-specified path, if defined `here <https://github.com/PyPSA/pypsa-eur-sec/blob/413254e241fb37f55b41caba7264644805ad8e97/config.default.yaml#L56>`_.
 The paper `Speed of technological transformations required in Europe to achieve different climate goals (2022) <https://doi.org/10.1016/j.joule.2022.04.016>`__ defines CO_2 budgets corresponding to global temperature increases (1.5C – 2C) as response to the emissions. Here, global carbon budgets are converted to European budgets assuming equal-per capita distribution which translates into a 6.43% share for Europe. The carbon budgets are in this paper distributed throughout the transition paths assuming an exponential decay. Emissions e(t) in every year t are limited by
 
 .. math::
-  e(t) = e_0 (1+ (r+m)t) e^(-mt)
+  e(t) = e_0 (1+ (r+m)t) e^{-mt}
 
 where r is the initial linear growth rate, which here is assumed to be r=0, and the decay parameter m is determined by imposing the integral of the path to be equal to the budget for Europe. Following this approach, the CO_2 budget is defined. Following the same approach as in this paper, add the following to the ``scenario.sector_opts``
 E.g.  ``-cb25.7ex0`` (1.5C increase)

doc/release_notes.rst

@@ -65,10 +65,14 @@ incorporates retrofitting options to hydrogen.
 
 * Add option for BtL (Biomass to liquid fuel/oil) with and without CC
 
+* Add option for minimum part load for Fischer-Tropsch plants (default: 90%) and methanolisation plants (default: 50%).
+
 * Units are assigned to the buses. These only provide a better understanding. The specifications of the units are not taken into account in the optimisation, which means that no automatic conversion of units takes place.
 
 * Option ``retrieve_sector_databundle`` to automatically retrieve and extract data bundle.
 
+* Add option to use waste heat of electrolysis in district heating networks (``use_electrolysis_waste_heat``).
+
 * Add regionalised hydrogen salt cavern storage potentials from `Technical Potential of Salt Caverns for Hydrogen Storage in Europe <https://doi.org/10.20944/preprints201910.0187.v1>`_.
 
 * Add option to sweep the global CO2 sequestration potentials with keyword ``seq200`` in the ``{sector_opts}`` wildcard (for limit of 200 Mt CO2).

doc/supply_demand.rst

@@ -427,7 +427,7 @@ We assume that the primary route can be replaced by a third route in 2050, using
    FeO + H_2  \xrightarrow{} Fe + H_2O
 
 
-This circumvents the process emissions associated with the use of coke. For hydrogen- based DRI, we assume energy requirements of 1.7 MWh :math:`_{H_2}` /t steel (Vogl et. al) <https://doi.org/10.1016/j.jclepro.2018.08.279>`_ and 0.322 MWh :math:`_{el}`/t steel `(HYBRIT 2016) <https://dh5k8ug1gwbyz.cloudfront.net/uploads/2021/02/Hybrit-broschure-engelska.pdf>`_. 
+This circumvents the process emissions associated with the use of coke. For hydrogen- based DRI, we assume energy requirements of 1.7 MWh :math:`_{H_2}` /t steel `(Vogl et. al) <https://doi.org/10.1016/j.jclepro.2018.08.279>`_ and 0.322 MWh :math:`_{el}`/t steel `(HYBRIT 2016) <https://dh5k8ug1gwbyz.cloudfront.net/uploads/2021/02/Hybrit-broschure-engelska.pdf>`_. 
 								
 					
 The share of steel produced via the primary route is exogenously set in the `config file <https://github.com/PyPSA/pypsa-eur-sec/blob/3daff49c9999ba7ca7534df4e587e1d516044fc3/config.default.yaml#L279>`_. The share of steel obtained via hydrogen-based DRI plus EAF is also set exogenously in the `config file <https://github.com/PyPSA/pypsa-eur-sec/blob/3daff49c9999ba7ca7534df4e587e1d516044fc3/config.default.yaml#L287>`_. The remaining share is manufactured through the secondary route using scrap metal in EAF. Bioenergy as alternative to coke in blast furnaces is not considered in the model (`Mandova et.al <https://doi.org/10.1016/j.biombioe.2018.04.021>`_, `Suopajärvi et.al <https://doi.org/10.1016/j.apenergy.2018.01.060>`_).
@@ -453,7 +453,7 @@ Statistics for the production of ammonia, which is commonly used as a fertilizer
 
 The Haber-Bosch process is not explicitly represented in the model, such that demand for ammonia enters the model as a demand for hydrogen ( 6.5 MWh :math:`_{H_2}` / t :math:`_{NH_3}` ) and electricity ( 1.17 MWh :math:`_{el}` /t :math:`_{NH_3}` ) (see `Wang et. al <https://doi.org/10.1016/j.joule.2018.04.017>`_). Today, natural gas dominates in Europe as the source for the hydrogen used in the Haber-Bosch process, but the model can choose among the various hydrogen supply options described in the hydrogen section (see :ref:`Hydrogen supply`)
 
-The total production and specific energy consumption of chlorine and methanol is taken from a `DECHEMA report <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry.pdf>`_. According to this source, the production of chlorine amounts to 9.58 MtCl/a, which is assumed to require electricity at 3.6 MWh `:math:`_{el}`/t of chlorine and yield hydrogen at 0.937 MWh :math:`_{H_2}`/t of chlorine in the chloralkali process. The production of methanol adds up to 1.5 MtMeOH/a, requiring electricity at 0.167 MWh :math:`_{el}`/t of methanol and methane at 10.25 MWh :math:`_{CH_4}`/t of methanol.
+The total production and specific energy consumption of chlorine and methanol is taken from a `DECHEMA report <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry.pdf>`_. According to this source, the production of chlorine amounts to 9.58 MtCl/a, which is assumed to require electricity at 3.6 MWh :math:`_{el}`/t of chlorine and yield hydrogen at 0.937 MWh :math:`_{H_2}`/t of chlorine in the chloralkali process. The production of methanol adds up to 1.5 MtMeOH/a, requiring electricity at 0.167 MWh :math:`_{el}`/t of methanol and methane at 10.25 MWh :math:`_{CH_4}`/t of methanol.
 		 	 	 		
 						
 The production of ammonia, methanol, and chlorine production is deducted from the JRC IDEES basic chemicals, leaving the production totals of high-value chemicals. For this, we assume that the liquid hydrocarbon feedstock comes from synthetic or fossil- origin naphtha (14 MWh :math:`_{naphtha}`/t of HVC, similar to `Lechtenböhmer et al <https://doi.org/10.1016/j.energy.2016.07.110>`_), ignoring the methanol-to-olefin route. Furthermore, we assume the following transformations of the energy-consuming processes in the production of plastics: the final energy consumption in steam processing is converted to methane since requires temperature above 500 °C (4.1 MWh :math:`_{CH_4}` /t of HVC, see `Rehfeldt et al. <https://doi.org/10.1007/s12053-017-9571-y>`_); and the remaining processes are electrified using the current efficiency of microwave for high-enthalpy heat processing, electric furnaces, electric process cooling and electric generic processes (2.85 MWh :math:`_{el}`/t of HVC).
@@ -461,7 +461,7 @@ The production of ammonia, methanol, and chlorine production is deducted from th
 The process emissions from feedstock in the chemical industry are as high as 0.369 t :math:`_{CO_2}`/t of ethylene equivalent. We consider process emissions for all the material output, which is a conservative approach since it assumes that all plastic-embedded :math:`CO_2` will eventually be released into the atmosphere. However, plastic disposal in landfilling will avoid, or at least delay, associated :math:`CO_2` emissions. 
 		 	 	 					
 Circular economy practices drastically reduce the amount of primary feedstock needed for the production of plastics in the model (see `Kullmann et al. <https://doi.org/10.1016/j.energy.2022.124660>`_, `Meys et al. (2021) <https://doi.org/10.1126/science.abg9853>`_, `Meys et al. (2020) <https://doi.org/10/gmxv6z>`_, `Gu et al. <https://doi.org/10/gf8n9w>`_) and consequently, also the energy demands and level of process emission. The percentage of plastics that are assumed to be mechanically recycled can be selected in the `config file <https://github.com/PyPSA/pypsa-eur-sec/blob/776596ab9ac6a6cc93422ccfd0383abeffb0baa9/config.default.yaml#L315>`_, as well as 
-the percentage that is chemically recycled, see `config file <https://github.com/PyPSA/pypsa-eur-sec/blob/776596ab9ac6a6cc93422ccfd0383abeffb0baa9/config.default.yaml#L316>`_ The energy consumption for those recycling processes are respectively 0.547 MWh :math:`_{el}`/t of HVC (as indicated in the `config file <https://github.com/PyPSA/pypsa-eur-sec/blob/776596ab9ac6a6cc93422ccfd0383abeffb0baa9/config.default.yaml#L318>`_) (`Meys et al. (2020) <https://doi.org/10/gmxv6z>`_), and 6.9 MWh :math:`_{el}`/t of HVC (as indicated in the config file `<https://github.com/PyPSA/pypsa-eur-sec/blob/776596ab9ac6a6cc93422ccfd0383abeffb0baa9/config.default.yaml#L319>`_) based on pyrolysis and electric steam cracking (see `Materials Economics <https://materialeconomics.com/publications/industrial-transformation-2050>`_ report). 
+the percentage that is chemically recycled, see `config file <https://github.com/PyPSA/pypsa-eur-sec/blob/776596ab9ac6a6cc93422ccfd0383abeffb0baa9/config.default.yaml#L316>`_ The energy consumption for those recycling processes are respectively 0.547 MWh :math:`_{el}`/t of HVC (as indicated in the `config file <https://github.com/PyPSA/pypsa-eur-sec/blob/776596ab9ac6a6cc93422ccfd0383abeffb0baa9/config.default.yaml#L318>`_) (`Meys et al. (2020) <https://doi.org/10/gmxv6z>`_), and 6.9 MWh :math:`_{el}`/t of HVC (as indicated in the `config file <https://github.com/PyPSA/pypsa-eur-sec/blob/776596ab9ac6a6cc93422ccfd0383abeffb0baa9/config.default.yaml#L319>`_) based on pyrolysis and electric steam cracking (see `Materials Economics <https://materialeconomics.com/publications/industrial-transformation-2050>`_ report). 
 
 
 **Non-metallic Mineral Products**
@@ -486,11 +486,11 @@ With the exception of electricity demand and biomass demand for low-temperature
 
 *Ceramics*
 
-The ceramics sector is assumed to be fully electrified based on the current efficiency of already electrified processes which include microwave drying and sintering of raw materials, electric kilns for primary production processes, electric furnaces for the `product finishing <https://data.europa.eu/doi/10.2760/182725>`_. In total, the final electricity consumption is 0.44 MWh/t of ceramic. The manufacturing of ceramics includes process emissions of 0.03 t :math:`_{CO_2} `/t of ceramic. For a detailed overview of the ceramics industry sector see `Furszyfer Del Rio et al <https://doi.org/10.1016/j.rser.2021.111885>`_.
+The ceramics sector is assumed to be fully electrified based on the current efficiency of already electrified processes which include microwave drying and sintering of raw materials, electric kilns for primary production processes, electric furnaces for the `product finishing <https://data.europa.eu/doi/10.2760/182725>`_. In total, the final electricity consumption is 0.44 MWh/t of ceramic. The manufacturing of ceramics includes process emissions of 0.03 t :math:`_{CO_2}`/t of ceramic. For a detailed overview of the ceramics industry sector see `Furszyfer Del Rio et al <https://doi.org/10.1016/j.rser.2021.111885>`_.
 
 *Glass*
 
-The production of glass is assumed to be fully electrified based on the current efficiency of electric melting tanks and electric annealing which adds up to an electricity demand of 2.07 MWh :math:`_{el}l/t` of `glass <https://doi.org/10/f9df2m>`_. The manufacturing of glass incurs process emissions of 0.1 t :math:`_{CO_2} `/t of glass. Potential efficiency improvements, which according to `Lechtenböhmer et al <https://doi.org/10/f9df2m>`_ could reduce energy demands to 0.85 MW :math:`_{el}`/t of glass, have not been considered. For a detailed overview of the glass industry sector see `Furszyfer Del Rio et al <https://doi.org/10.1016/j.rser.2021.111885>`_.
+The production of glass is assumed to be fully electrified based on the current efficiency of electric melting tanks and electric annealing which adds up to an electricity demand of 2.07 MWh :math:`_{el}`/t of `glass <https://doi.org/10/f9df2m>`_. The manufacturing of glass incurs process emissions of 0.1 t :math:`_{CO_2}`/t of glass. Potential efficiency improvements, which according to `Lechtenböhmer et al <https://doi.org/10/f9df2m>`_ could reduce energy demands to 0.85 MW :math:`_{el}`/t of glass, have not been considered. For a detailed overview of the glass industry sector see `Furszyfer Del Rio et al <https://doi.org/10.1016/j.rser.2021.111885>`_.
 
 
 **Non-ferrous Metals**

scripts/add_existing_baseyear.py

@@ -12,7 +12,7 @@ import xarray as xr
 import pypsa
 import yaml
 
-from prepare_sector_network import prepare_costs, define_spatial
+from prepare_sector_network import prepare_costs, define_spatial, cluster_heat_buses
 from helper import override_component_attrs, update_config_with_sector_opts
 
 from types import SimpleNamespace
@@ -563,5 +563,9 @@ if __name__ == "__main__":
         add_heating_capacities_installed_before_baseyear(n, baseyear, grouping_years_heat,
                                                          ashp_cop, gshp_cop, time_dep_hp_cop, costs, default_lifetime)
 
+    if options.get("cluster_heat_buses", False):
+        cluster_heat_buses(n)
+
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
+
     n.export_to_netcdf(snakemake.output[0])

scripts/prepare_sector_network.py

@@ -19,6 +19,7 @@ from helper import override_component_attrs, generate_periodic_profiles, update_
 from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation
 from networkx.algorithms import complement
 from pypsa.geo import haversine_pts
+from pypsa.io import import_components_from_dataframe
 
 import logging
 logger = logging.getLogger(__name__)
@@ -26,6 +27,9 @@ logger = logging.getLogger(__name__)
 from types import SimpleNamespace
 spatial = SimpleNamespace()
 
+from packaging.version import Version, parse
+pd_version = parse(pd.__version__)
+agg_group_kwargs = dict(numeric_only=False) if pd_version >= Version("1.3") else {}
 
 def define_spatial(nodes, options):
     """
@@ -1029,7 +1033,7 @@ def add_storage_and_grids(n, costs):
         )
 
     # hydrogen stored overground (where not already underground)
-    h2_capital_cost = costs.at["hydrogen storage tank incl. compressor", "fixed"]
+    h2_capital_cost = costs.at["hydrogen storage tank type 1 including compressor", "fixed"]
     nodes_overground = h2_caverns.index.symmetric_difference(nodes)
 
     n.madd("Store",
@@ -2259,6 +2263,7 @@ def add_industry(n, costs):
             bus3=spatial.co2.nodes,
             carrier="methanolisation",
             p_nom_extendable=True,
+            p_min_pu=options.get("min_part_load_methanolisation", 0),
             capital_cost=costs.at["methanolisation", 'fixed'] * options["MWh_MeOH_per_MWh_H2"], # EUR/MW_H2/a
             lifetime=costs.at["methanolisation", 'lifetime'],
             efficiency=options["MWh_MeOH_per_MWh_H2"],
@@ -2366,6 +2371,7 @@ def add_industry(n, costs):
         capital_cost=costs.at["Fischer-Tropsch", 'fixed'] * costs.at["Fischer-Tropsch", 'efficiency'], # EUR/MW_H2/a
         efficiency2=-costs.at["oil", 'CO2 intensity'] * costs.at["Fischer-Tropsch", 'efficiency'],
         p_nom_extendable=True,
+        p_min_pu=options.get("min_part_load_fischer_tropsch", 0),
         lifetime=costs.at['Fischer-Tropsch', 'lifetime']
     )
 
@@ -2508,6 +2514,11 @@ def add_waste_heat(n):
             n.links.loc[urban_central + " Fischer-Tropsch", "bus3"] = urban_central + " urban central heat"
             n.links.loc[urban_central + " Fischer-Tropsch", "efficiency3"] = 0.95 - n.links.loc[urban_central + " Fischer-Tropsch", "efficiency"]
 
+        # TODO integrate useable waste heat efficiency into technology-data from DEA
+        if options.get('use_electrolysis_waste_heat', False):
+            n.links.loc[urban_central + " H2 Electrolysis", "bus2"] = urban_central + " urban central heat"
+            n.links.loc[urban_central + " H2 Electrolysis", "efficiency2"] = 0.84 - n.links.loc[urban_central + " H2 Electrolysis", "efficiency"]
+
         if options['use_fuel_cell_waste_heat']:
             n.links.loc[urban_central + " H2 Fuel Cell", "bus2"] = urban_central + " urban central heat"
             n.links.loc[urban_central + " H2 Fuel Cell", "efficiency2"] = 0.95 - n.links.loc[urban_central + " H2 Fuel Cell", "efficiency"]
@@ -2635,6 +2646,98 @@ def limit_individual_line_extension(n, maxext):
     n.links.loc[hvdc, 'p_nom_max'] = n.links.loc[hvdc, 'p_nom'] + maxext
 
 
+aggregate_dict = {
+    "p_nom": "sum",
+    "s_nom": "sum",
+    "v_nom": "max",
+    "v_mag_pu_max": "min",
+    "v_mag_pu_min": "max",
+    "p_nom_max": "sum",
+    "s_nom_max": "sum",
+    "p_nom_min": "sum",
+    "s_nom_min": "sum",
+    'v_ang_min': "max",
+    "v_ang_max":"min",
+    "terrain_factor":"mean",
+    "num_parallel": "sum",
+    "p_set": "sum",
+    "e_initial": "sum",
+    "e_nom": "sum",
+    "e_nom_max": "sum",
+    "e_nom_min": "sum",
+    "state_of_charge_initial": "sum",
+    "state_of_charge_set": "sum",
+    "inflow": "sum",
+    "p_max_pu": "first",
+    "x": "mean",
+    "y": "mean"
+}
+
+def cluster_heat_buses(n):
+    """Cluster residential and service heat buses to one representative bus.
+    This can be done to save memory and speed up optimisation
+    """
+
+    def define_clustering(attributes, aggregate_dict):
+        """Define how attributes should be clustered.
+        Input:
+            attributes    : pd.Index()
+            aggregate_dict: dictionary (key: name of attribute, value
+                                        clustering method)
+
+        Returns:
+            agg           : clustering dictionary
+        """
+        keys = attributes.intersection(aggregate_dict.keys())
+        agg = dict(
+            zip(
+                attributes.difference(keys),
+                ["first"] * len(df.columns.difference(keys)),
+            )
+        )
+        for key in keys:
+            agg[key] = aggregate_dict[key]
+        return agg
+
+    logger.info("Cluster residential and service heat buses.")
+    components = ["Bus", "Carrier", "Generator", "Link", "Load", "Store"]
+
+    for c in n.iterate_components(components):
+        df = c.df
+        cols = df.columns[df.columns.str.contains("bus") | (df.columns=="carrier")]
+
+        # rename columns and index
+        df[cols] = (df[cols]
+                         .apply(lambda x: x.str.replace("residential ","")
+                                .str.replace("services ", ""), axis=1))
+        df = df.rename(index=lambda x: x.replace("residential ","")
+                       .replace("services ", ""))
+
+
+        # cluster heat nodes
+        # static dataframe
+        agg = define_clustering(df.columns, aggregate_dict)
+        df = df.groupby(level=0).agg(agg, **agg_group_kwargs)
+        # time-varying data
+        pnl = c.pnl
+        agg = define_clustering(pd.Index(pnl.keys()), aggregate_dict)
+        for k in pnl.keys():
+            pnl[k].rename(columns=lambda x: x.replace("residential ","")
+                           .replace("services ", ""), inplace=True)
+            pnl[k] = (
+                 pnl[k]
+                .groupby(level=0, axis=1)
+                .agg(agg[k], **agg_group_kwargs)
+            )
+
+        # remove unclustered assets of service/residential
+        to_drop = c.df.index.difference(df.index)
+        n.mremove(c.name, to_drop)
+        # add clustered assets
+        to_add = df.index.difference(c.df.index)
+        import_components_from_dataframe(n, df.loc[to_add], c.name)
+
+
 def apply_time_segmentation(n, segments, solver_name="cbc",
                             overwrite_time_dependent=True):
     """Aggregating time series to segments with different lengths
@@ -2716,6 +2819,7 @@ def set_temporal_aggregation(n, opts, solver_name):
             n = apply_time_segmentation(n, segments, solver_name=solver_name)
             break
     return n
+
 #%%
 if __name__ == "__main__":
     if 'snakemake' not in globals():
@@ -2863,5 +2967,13 @@ if __name__ == "__main__":
     if options['electricity_grid_connection']:
         add_electricity_grid_connection(n, costs)
 
+    first_year_myopic = ((snakemake.config["foresight"] == 'myopic') and
+                         (snakemake.config["scenario"]["planning_horizons"][0]==investment_year))
+
+    if options.get("cluster_heat_buses", False) and not first_year_myopic:
+        cluster_heat_buses(n)
+
+
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
+
     n.export_to_netcdf(snakemake.output[0])

test/config.myopic.yaml

@@ -3,10 +3,10 @@ version: 0.6.0
 logging_level: INFO
 
 retrieve_sector_databundle: true
+retrieve_cost_data: true
 
 results_dir: results/
 summary_dir: results
-costs_dir: ../technology-data/outputs/
 run: test-myopic  # use this to keep track of runs with different settings
 foresight: myopic # options are overnight, myopic, perfect (perfect is not yet implemented)
 # if you use myopic or perfect foresight, set the investment years in "planning_horizons" below
@@ -61,7 +61,7 @@ snapshots:
   # arguments to pd.date_range
   start: "2013-03-01"
   end: "2013-04-01"
-  closed: left # end is not inclusive
+  inclusive: left # end is not inclusive
 
 atlite:
   cutout: ../pypsa-eur/cutouts/be-03-2013-era5.nc
@@ -320,6 +320,7 @@ industry:
 
 costs:
   year: 2030
+  version: v0.5.0
   lifetime: 25 #default lifetime
   # From a Lion Hirth paper, also reflects average of Noothout et al 2016
   discountrate: 0.07

test/config.overnight.yaml

@@ -3,10 +3,10 @@ version: 0.6.0
 logging_level: INFO
 
 retrieve_sector_databundle: true
+retrieve_cost_data: true
 
 results_dir: results/
 summary_dir: results
-costs_dir: ../technology-data/outputs/
 run: test-overnight  # use this to keep track of runs with different settings
 foresight: overnight # options are overnight, myopic, perfect (perfect is not yet implemented)
 # if you use myopic or perfect foresight, set the investment years in "planning_horizons" below
@@ -59,7 +59,7 @@ snapshots:
   # arguments to pd.date_range
   start: "2013-03-01"
   end: "2013-04-01"
-  closed: left # end is not inclusive
+  inclusive: left # end is not inclusive
 
 atlite:
   cutout: ../pypsa-eur/cutouts/be-03-2013-era5.nc
@@ -318,6 +318,7 @@ industry:
 
 costs:
   year: 2030
+  version: v0.5.0
   lifetime: 25 #default lifetime
   # From a Lion Hirth paper, also reflects average of Noothout et al 2016
   discountrate: 0.07