Merge branch 'master' into introduce_modularity_clustering

Snakefile

@@ -13,7 +13,7 @@ if not exists("config.yaml"):
 
 configfile: "config.yaml"
 
-COSTS="data/costs.csv"
+COSTS="resources/costs.csv"
 ATLITE_NPROCESSES = config['atlite'].get('nprocesses', 4)
 
 wildcard_constraints:
@@ -78,7 +78,6 @@ rule build_load_data:
     log: "logs/build_load_data.log"
     script: 'scripts/build_load_data.py'
 
-
 rule build_powerplants:
     input:
         base_network="networks/base.nc",
@@ -163,6 +162,11 @@ if config['enable'].get('retrieve_cutout', True):
         output: "cutouts/{cutout}.nc"
         run: move(input[0], output[0])
 
+if config['enable'].get('retrieve_cost_data', True):
+    rule retrieve_cost_data:
+        input: HTTP.remote(f"raw.githubusercontent.com/PyPSA/technology-data/{config['costs']['version']}/outputs/costs_{config['costs']['year']}.csv", keep_local=True)
+        output: COSTS
+        run: move(input[0], output[0])
 
 if config['enable'].get('build_natura_raster', False):
     rule build_natura_raster:

config.default.yaml

@@ -25,6 +25,7 @@ snapshots:
 enable:
   prepare_links_p_nom: false
   retrieve_databundle: true
+  retrieve_cost_data: true
   build_cutout: false
   retrieve_cutout: true
   build_natura_raster: false
@@ -52,7 +53,7 @@ electricity:
     Generator: [solar, onwind, offwind-ac, offwind-dc, OCGT]
     StorageUnit: [] # battery, H2
     Store: [battery, H2]
-    Link: [AC, DC]
+    Link: [] # H2 pipeline
 
   # use pandas query strings here, e.g. Country not in ['Germany']
   powerplants_filter: (DateOut >= 2022 or DateOut != DateOut)
@@ -217,9 +218,18 @@ load:
 
 costs:
   year: 2030
-  discountrate: 0.07 # From a Lion Hirth paper, also reflects average of Noothout et al 2016
+  version: v0.1.0
-  USD2013_to_EUR2013: 0.7532 # [EUR/USD] ECB: https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html
+  rooftop_share: 0.14  # based on the potentials, assuming  (0.1 kW/m2 and 10 m2/person)
-  marginal_cost: # EUR/MWh
+  fill_values:
+    FOM: 0
+    VOM: 0
+    efficiency: 1
+    fuel: 0
+    investment: 0
+    lifetime: 25
+    "CO2 intensity": 0
+    "discount rate": 0.07
+  marginal_cost:
     solar: 0.01
     onwind: 0.015
     offwind: 0.015

config.tutorial.yaml

@@ -26,6 +26,7 @@ snapshots:
 enable:
   prepare_links_p_nom: false
   retrieve_databundle: true
+  retrieve_cost_data: true
   build_cutout: false
   retrieve_cutout: true
   build_natura_raster: false
@@ -40,7 +41,7 @@ electricity:
     Generator: [OCGT]
     StorageUnit: [] #battery, H2
     Store: [battery, H2]
-    Link: []
+    Link: [] # H2 pipeline
 
   max_hours:
     battery: 6
@@ -154,8 +155,17 @@ load:
 
 costs:
   year: 2030
-  discountrate: 0.07 # From a Lion Hirth paper, also reflects average of Noothout et al 2016
+  version: v0.1.0
-  USD2013_to_EUR2013: 0.7532 # [EUR/USD] ECB: https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html
+  rooftop_share: 0.14
+  fill_values:
+    FOM: 0
+    VOM: 0
+    efficiency: 1
+    fuel: 0
+    investment: 0
+    lifetime: 25
+    "CO2 intensity": 0
+    "discount rate": 0.07
   marginal_cost:
     solar: 0.01
     onwind: 0.015

doc/configtables/costs.csv

@@ -1,8 +1,9 @@
 ,Unit,Values,Description
-year,--,"YYYY; e.g. '2030'","Year for which to retrieve cost assumptions of ``data/costs.csv``."
+year,--,"YYYY; e.g. '2030'","Year for which to retrieve cost assumptions of ``resources/costs.csv``."
-discountrate,--,float,"Default discount rate if not specified for a technology in ``data/costs.csv``."
+version,--,"vX.X.X; e.g. 'v0.1.0'","Version of ``technology-data`` repository to use."
-USD2013_to_EUR2013,--,float,"Exchange rate from USD :math:`_{2013}` to EUR :math:`_{2013}` from `ECB <https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html>`_"
+rooftop_share,--,float,"Share of rooftop PV when calculating capital cost of solar (joint rooftop and utility-scale PV)."
-capital_cost,EUR/MW,"Keys should be in the 'technology' column of ``data/costs.csv``. Values can be any float.","For the given technologies, assumptions about their capital investment costs are set to the corresponding value. Optional; overwrites cost assumptions from ``data/costs.csv``."
+fill_values,--,float,"Default values if not specified for a technology in ``resources/costs.csv``."
-marginal_cost,EUR/MWh,"Keys should be in the 'technology' column of ``data/costs.csv``. Values can be any float.","For the given technologies, assumptions about their marginal operating costs are set to the corresponding value. Optional; overwrites cost assumptions from ``data/costs.csv``."
+capital_cost,EUR/MW,"Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.","For the given technologies, assumptions about their capital investment costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``."
+marginal_cost,EUR/MWh,"Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.","For the given technologies, assumptions about their marginal operating costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``."
 emission_prices,,,"Specify exogenous prices for emission types listed in ``network.carriers`` to marginal costs."
 -- co2,EUR/t,float,"Exogenous price of carbon-dioxide added to the marginal costs of fossil-fuelled generators according to their carbon intensity. Added through the keyword ``Ep`` in the ``{opts}`` wildcard only in the rule :mod:`prepare_network``."

doc/configuration.rst

@@ -265,8 +265,8 @@ Define additional generator attribute for conventional carrier types. If a scala
    :file: configtables/costs.csv
 
 .. note::
-    To change cost assumptions in more detail (i.e. other than ``marginal_cost`` and ``capital_cost``), consider modifying cost assumptions directly in ``data/costs.csv`` as this is not yet supported through the config file.
+    To change cost assumptions in more detail (i.e. other than ``marginal_cost`` and ``capital_cost``), consider modifying cost assumptions directly in ``resources/costs.csv`` as this is not yet supported through the config file.
-    You can also build multiple different cost databases. Make a renamed copy of ``data/costs.csv`` (e.g. ``data/costs-optimistic.csv``) and set the variable ``COSTS=data/costs-optimistic.csv`` in the ``Snakefile``.
+    You can also build multiple different cost databases. Make a renamed copy of ``resources/costs.csv`` (e.g. ``data/costs-optimistic.csv``) and set the variable ``COSTS=data/costs-optimistic.csv`` in the ``Snakefile``.
 
 .. _solving_cf:
 

doc/costs.rst

@@ -7,7 +7,9 @@
 Cost Assumptions
 ##################
 
-The database of cost assumptions is stored in ``data/costs.csv``.
+The database of cost assumptions is retrieved from the repository `PyPSA/technology-data <https://github.com/pypsa/technology-data>`_ and then saved to``resources/costs.csv``. Cost assumptions of previous PyPSA-Eur versions can be restored by setting in the ``Snakefile``: ``COSTS="data/costs.csv".
+
+The ``config.yaml`` provides options to choose a reference year (``costs: year:``) and use a specific version of the repository ``costs: version:``.
 
 It includes cost assumptions for all included technologies for specific
 years from various sources, namely for
@@ -39,15 +41,6 @@ Modifying Cost Assumptions
 
 Some cost assumptions (e.g. marginal cost and capital cost) can be directly overwritten in the ``config.yaml`` (cf. Section  :ref:`costs_cf`  in :ref:`config`).
 
-To change cost assumptions in more detail, modify cost assumptions directly in ``data/costs.csv`` as this is not yet supported through the config file.
+To change cost assumptions in more detail, modify cost assumptions directly in ``resources/costs.csv`` as this is not yet supported through the config file.
 
-You can also build multiple different cost databases. Make a renamed copy of ``data/costs.csv`` (e.g. ``data/costs-optimistic.csv``) and set the variable ``COSTS=data/costs-optimistic.csv`` in the ``Snakefile``.
+You can also build multiple different cost databases. Make a renamed copy of ``resources/costs.csv`` (e.g. ``data/costs-optimistic.csv``) and set the variable ``COSTS=data/costs-optimistic.csv`` in the ``Snakefile``.
-
-
-Default Cost Assumptions
-========================
-
-.. csv-table::
-   :header-rows: 1
-   :widths: 10,3,5,4,6,8
-   :file: ../data/costs.csv

doc/release_notes.rst

@@ -92,8 +92,14 @@ Upcoming Release
 
 * Hierarchical clustering was introduced. Distance metric is calculated from renewable potentials on hourly (feature entry ends with `-time`) or annual (feature entry in config end with `-cap`) values.
 
+
 * Greedy modularity clustering was introduced. Distance metric is based on electrical distance taking into account the impedance of all transmission lines of the network.
 
+* Techno-economic parameters of technologies (e.g. costs and efficiencies) will now be retrieved from a separate repository `PyPSA/technology-data <https://github.com/pypsa/technology-data>`_ 
+  that collects assumptions from a variety of sources. It is activated by default with ``enable: retrieve_cost_data: true`` and controlled with ``costs: year:`` and ``costs: version:``. 
+  The location of this data changed from ``data/costs.csv`` to ``resources/costs.csv`` 
+  [`#184 <https://github.com/PyPSA/pypsa-eur/pull/184>`_].
+
 
 Synchronisation Release - Ukraine and Moldova (17th March 2022)
 ===============================================================
@@ -245,7 +251,6 @@ PyPSA-Eur 0.4.0 (22th September 2021)
   PyPSA network solving functions were not told about the solver logfile specified
   in the Snakemake file [`#247 <https://github.com/PyPSA/pypsa-eur/pull/247>`_]
 
-
 PyPSA-Eur 0.3.0 (7th December 2020)
 ===================================
 

doc/tutorial.rst

@@ -216,7 +216,7 @@ A job (here ``simplify_network``) will display its attributes and normally some
 
     [<DATETIME>]
     rule simplify_network:
-        input: networks/elec.nc, data/costs.csv, resources/regions_onshore.geojson, resources/regions_offshore.geojson
+        input: networks/elec.nc, resources/costs.csv, resources/regions_onshore.geojson, resources/regions_offshore.geojson
         output: networks/elec_s.nc, resources/regions_onshore_elec_s.geojson, resources/regions_offshore_elec_s.geojson, resources/clustermaps_elec_s.h5
         jobid: 3
         benchmark: benchmarks/simplify_network/elec_s

envs/environment.yaml

@@ -53,6 +53,8 @@ dependencies:
   - tqdm
   - pytz
   - tabula-py
+  - mergedeep
+  - pyxlsb
 
   - pip:
     - vresutils>=0.3.1

scripts/add_electricity.py

@@ -13,7 +13,7 @@ Relevant Settings
 
     costs:
         year:
-        USD2013_to_EUR2013:
+        version:
         dicountrate:
         emission_prices:
 
@@ -46,7 +46,7 @@ Relevant Settings
 Inputs
 ------
 
-- ``data/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity.
+- ``resources/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity.
 - ``data/bundle/hydro_capacities.csv``: Hydropower plant store/discharge power capacities, energy storage capacity, and average hourly inflow by country.
 
     .. image:: ../img/hydrocapacities.png
@@ -93,7 +93,6 @@ import xarray as xr
 import geopandas as gpd
 import powerplantmatching as pm
 from powerplantmatching.export import map_country_bus
-
 from vresutils import transfer as vtransfer
 
 idx = pd.IndexSlice
@@ -131,23 +130,14 @@ def _add_missing_carriers_from_costs(n, costs, carriers):
 def load_costs(tech_costs, config, elec_config, Nyears=1.):
 
     # set all asset costs and other parameters
-    costs = pd.read_csv(tech_costs, index_col=list(range(3))).sort_index()
+    costs = pd.read_csv(tech_costs, index_col=[0,1]).sort_index()
 
-    # correct units to MW and EUR
+    # correct units to MW
     costs.loc[costs.unit.str.contains("/kW"),"value"] *= 1e3
-    costs.loc[costs.unit.str.contains("USD"),"value"] *= config['USD2013_to_EUR2013']
+    costs.unit = costs.unit.str.replace("/kW", "/MW")
 
-    costs = (costs.loc[idx[:,config['year'],:], "value"]
+    fill_values = config["fill_values"]
-             .unstack(level=2).groupby("technology").sum(min_count=1))
+    costs = costs.value.unstack().fillna(fill_values)
-
-    costs = costs.fillna({"CO2 intensity" : 0,
-                          "FOM" : 0,
-                          "VOM" : 0,
-                          "discount rate" : config['discountrate'],
-                          "efficiency" : 1,
-                          "fuel" : 0,
-                          "investment" : 0,
-                          "lifetime" : 25})
 
     costs["capital_cost"] = ((calculate_annuity(costs["lifetime"], costs["discount rate"]) +
                              costs["FOM"]/100.) *
@@ -163,8 +153,8 @@ def load_costs(tech_costs, config, elec_config, Nyears=1.):
     costs.at['OCGT', 'co2_emissions'] = costs.at['gas', 'co2_emissions']
     costs.at['CCGT', 'co2_emissions'] = costs.at['gas', 'co2_emissions']
 
-    costs.at['solar', 'capital_cost'] = 0.5*(costs.at['solar-rooftop', 'capital_cost'] +
+    costs.at['solar', 'capital_cost'] = config["rooftop_share"] * costs.at['solar-rooftop', 'capital_cost'] + \
-                                             costs.at['solar-utility', 'capital_cost'])
+                                        (1-config["rooftop_share"]) * costs.at['solar-utility', 'capital_cost']
 
     def costs_for_storage(store, link1, link2=None, max_hours=1.):
         capital_cost = link1['capital_cost'] + max_hours * store['capital_cost']
@@ -179,7 +169,7 @@ def load_costs(tech_costs, config, elec_config, Nyears=1.):
         costs_for_storage(costs.loc["battery storage"], costs.loc["battery inverter"],
                           max_hours=max_hours['battery'])
     costs.loc["H2"] = \
-        costs_for_storage(costs.loc["hydrogen storage"], costs.loc["fuel cell"],
+        costs_for_storage(costs.loc["hydrogen storage underground"], costs.loc["fuel cell"],
                           costs.loc["electrolysis"], max_hours=max_hours['H2'])
 
     for attr in ('marginal_cost', 'capital_cost'):

scripts/add_extra_components.py

@@ -13,7 +13,7 @@ Relevant Settings
 
     costs:
         year:
-        USD2013_to_EUR2013:
+        version:
         dicountrate:
         emission_prices:
 
@@ -32,7 +32,7 @@ Relevant Settings
 Inputs
 ------
 
-- ``data/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity.
+- ``resources/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity.
 
 Outputs
 -------
@@ -76,16 +76,19 @@ def attach_storageunits(n, costs, elec_opts):
     lookup_dispatch = {"H2": "fuel cell", "battery": "battery inverter"}
 
     for carrier in carriers:
+        roundtrip_correction = 0.5 if carrier == "battery" else 1
+        
         n.madd("StorageUnit", buses_i, ' ' + carrier,
                bus=buses_i,
                carrier=carrier,
                p_nom_extendable=True,
                capital_cost=costs.at[carrier, 'capital_cost'],
                marginal_cost=costs.at[carrier, 'marginal_cost'],
-               efficiency_store=costs.at[lookup_store[carrier], 'efficiency'],
+               efficiency_store=costs.at[lookup_store[carrier], 'efficiency']**roundtrip_correction,
-               efficiency_dispatch=costs.at[lookup_dispatch[carrier], 'efficiency'],
+               efficiency_dispatch=costs.at[lookup_dispatch[carrier], 'efficiency']**roundtrip_correction,
                max_hours=max_hours[carrier],
-               cyclic_state_of_charge=True)
+               cyclic_state_of_charge=True
+        )
 
 
 def attach_stores(n, costs, elec_opts):
@@ -104,7 +107,7 @@ def attach_stores(n, costs, elec_opts):
                carrier='H2',
                e_nom_extendable=True,
                e_cyclic=True,
-               capital_cost=costs.at["hydrogen storage", "capital_cost"])
+               capital_cost=costs.at["hydrogen storage underground", "capital_cost"])
 
         n.madd("Link", h2_buses_i + " Electrolysis",
                bus0=buses_i,
@@ -140,7 +143,8 @@ def attach_stores(n, costs, elec_opts):
                bus0=buses_i,
                bus1=b_buses_i,
                carrier='battery charger',
-               efficiency=costs.at['battery inverter', 'efficiency'],
+               # the efficiencies are "round trip efficiencies"
+               efficiency=costs.at['battery inverter', 'efficiency']**0.5,
                capital_cost=costs.at['battery inverter', 'capital_cost'],
                p_nom_extendable=True,
                marginal_cost=costs.at["battery inverter", "marginal_cost"])
@@ -149,7 +153,7 @@ def attach_stores(n, costs, elec_opts):
                bus0=b_buses_i,
                bus1=buses_i,
                carrier='battery discharger',
-               efficiency=costs.at['battery inverter','efficiency'],
+               efficiency=costs.at['battery inverter','efficiency']**0.5,
                p_nom_extendable=True,
                marginal_cost=costs.at["battery inverter", "marginal_cost"])
 

scripts/build_hydro_profile.py

@@ -76,10 +76,32 @@ def get_eia_annual_hydro_generation(fn, countries):
     df = pd.read_csv(fn, skiprows=2, index_col=1, na_values=[u' ','--']).iloc[1:, 1:]
     df.index = df.index.str.strip()
 
+    former_countries = {
+        "Former Czechoslovakia": dict(
+            countries=["Czech Republic", "Slovakia"],
+            start=1980, end=1992),
+        "Former Serbia and Montenegro": dict(
+            countries=["Serbia", "Montenegro"],
+            start=1992, end=2005),
+        "Former Yugoslavia": dict(
+            countries=["Slovenia", "Croatia", "Bosnia and Herzegovina", "Serbia", "Montenegro", "North Macedonia"],
+            start=1980, end=1991),
+    }
+
+    for k, v in former_countries.items():
+        period = [str(i) for i in range(v["start"], v["end"]+1)]
+        ratio = df.loc[v['countries']].T.dropna().sum()
+        ratio /= ratio.sum()
+        for country in v['countries']:
+            df.loc[country, period] = df.loc[k, period] * ratio[country]
+
+    baltic_states = ["Latvia", "Estonia", "Lithuania"]
+    df.loc[baltic_states] = df.loc[baltic_states].T.fillna(df.loc[baltic_states].mean(axis=1)).T
+
     df.loc["Germany"] = df.filter(like='Germany', axis=0).sum()
-    df.loc["Serbia"] += df.loc["Kosovo"]
+    df.loc["Serbia"] += df.loc["Kosovo"].fillna(0.)
     df = df.loc[~df.index.str.contains('Former')]
-    df.drop(["Europe", "Germany, West", "Germany, East"], inplace=True)
+    df.drop(["Europe", "Germany, West", "Germany, East", "Kosovo"], inplace=True)
 
     df.index = cc.convert(df.index, to='iso2')
     df.index.name = 'countries'

scripts/make_summary.py

@@ -11,8 +11,9 @@ Relevant Settings
 .. code:: yaml
 
     costs:
-        USD2013_to_EUR2013:
+        year:
-        discountrate:
+        version:
+        fill_values:
         marginal_cost:
         capital_cost:
 

scripts/prepare_network.py

@@ -20,9 +20,10 @@ Relevant Settings
 .. code:: yaml
 
     costs:
+        year:
+        version:
+        fill_values:
         emission_prices:
-        USD2013_to_EUR2013:
-        discountrate:
         marginal_cost:
         capital_cost:
 
@@ -37,7 +38,7 @@ Relevant Settings
 Inputs
 ------
 
-- ``data/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity.
+- ``resources/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity.
 - ``networks/elec_s{simpl}_{clusters}.nc``: confer :ref:`cluster`
 
 Outputs

scripts/simplify_network.py

@@ -19,8 +19,9 @@ Relevant Settings
       aggregation_strategies:
 
     costs:
-        USD2013_to_EUR2013:
+        year:
-        discountrate:
+        version:
+        fill_values:
         marginal_cost:
         capital_cost:
 
@@ -45,7 +46,7 @@ Relevant Settings
 Inputs
 ------
 
-- ``data/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity.
+- ``resources/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity.
 - ``resources/regions_onshore.geojson``: confer :ref:`busregions`
 - ``resources/regions_offshore.geojson``: confer :ref:`busregions`
 - ``networks/elec.nc``: confer :ref:`electricity`

test/config.test1.yaml

@@ -25,6 +25,7 @@ snapshots:
 enable:
   prepare_links_p_nom: false
   retrieve_databundle: true
+  retrieve_cost_data: true
   build_cutout: false
   retrieve_cutout: true
   build_natura_raster: false
@@ -152,8 +153,17 @@ load:
 
 costs:
   year: 2030
-  discountrate: 0.07 # From a Lion Hirth paper, also reflects average of Noothout et al 2016
+  version: v0.1.0
-  USD2013_to_EUR2013: 0.7532 # [EUR/USD] ECB: https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html
+  rooftop_share: 0.14
+  fill_values:
+    FOM: 0
+    VOM: 0
+    efficiency: 1
+    fuel: 0
+    investment: 0
+    lifetime: 25
+    "CO2 intensity": 0
+    "discount rate": 0.07
   marginal_cost:
     solar: 0.01
     onwind: 0.015