Merge branch 'master' into meta

2022-07-26 14:02:26 +02:00 · 2022-07-26 14:02:26 +02:00 · 31bc87adba
commit 31bc87adba
parent 217780263d 84c8a9bc0e
21 changed files with 206 additions and 150 deletions
--- a/8
+++ b/8
@ -13,7 +13,7 @@ if not exists("config.yaml"):

 configfile: "config.yaml"

-COSTS="data/costs.csv"
+COSTS="resources/costs.csv"
 ATLITE_NPROCESSES = config['atlite'].get('nprocesses', 4)

 wildcard_constraints:
@ -77,7 +77,6 @@ rule build_load_data:
    output: "resources/load.csv"
    log: "logs/build_load_data.log"
    script: 'scripts/build_load_data.py'
-    

 rule build_powerplants:
    input:
@ -163,6 +162,11 @@ if config['enable'].get('retrieve_cutout', True):
        output: "cutouts/{cutout}.nc"
        run: move(input[0], output[0])

+if config['enable'].get('retrieve_cost_data', True):
+    rule retrieve_cost_data:
+        input: HTTP.remote(f"raw.githubusercontent.com/PyPSA/technology-data/{config['costs']['version']}/outputs/costs_{config['costs']['year']}.csv", keep_local=True)
+        output: COSTS
+        run: move(input[0], output[0])

 if config['enable'].get('build_natura_raster', False):
    rule build_natura_raster:
--- a/config.default.yaml
+++ b/config.default.yaml
@ -17,25 +17,6 @@ scenario:

 countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK']

-clustering:
-  simplify_network:
-    to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections)
-    algorithm: kmeans # choose from: [hac, kmeans]
-    feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc.
-  cluster_network:
-    algorithm: kmeans
-    feature: solar+onwind-time
-  aggregation_strategies:
-    generators:
-      p_nom_max: sum # use "min" for more conservative assumptions
-      p_nom_min: sum
-      p_min_pu: mean
-      marginal_cost: mean
-      committable: any
-      ramp_limit_up: max
-      ramp_limit_down: max
-      efficiency: mean
-
 snapshots:
  start: "2013-01-01"
  end: "2014-01-01"
@ -44,6 +25,7 @@ snapshots:
 enable:
  prepare_links_p_nom: false
  retrieve_databundle: true
+  retrieve_cost_data: true
  build_cutout: false
  retrieve_cutout: true
  build_natura_raster: false
@ -71,7 +53,7 @@ electricity:
    Generator: [solar, onwind, offwind-ac, offwind-dc, OCGT]
    StorageUnit: [] # battery, H2
    Store: [battery, H2]
-    Link: [AC, DC]
+    Link: [] # H2 pipeline

  # use pandas query strings here, e.g. Country not in ['Germany']
  powerplants_filter: (DateOut >= 2022 or DateOut != DateOut)
@ -101,27 +83,27 @@ atlite:
  nprocesses: 4
  cutouts:
    # use 'base' to determine geographical bounds and time span from config
-    # base: 
-      # module: era5  
+    # base:
+      # module: era5
    europe-2013-era5:
-      module: era5 # in priority order 
+      module: era5 # in priority order
      x: [-12., 35.]
      y: [33., 72]
      dx: 0.3
      dy: 0.3
      time: ['2013', '2013']
    europe-2013-sarah:
-      module: [sarah, era5] # in priority order 
+      module: [sarah, era5] # in priority order
      x: [-12., 45.]
      y: [33., 65]
      dx: 0.2
      dy: 0.2
      time: ['2013', '2013']
      sarah_interpolate: false
-      sarah_dir: 
+      sarah_dir:
      features: [influx, temperature]

-      
+
 renewable:
  onwind:
    cutout: europe-2013-era5
@ -230,15 +212,24 @@ transformers:
 load:
  power_statistics: True # only for files from <2019; set false in order to get ENTSOE transparency data 
  interpolate_limit: 3 # data gaps up until this size are interpolated linearly
-  time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from 
+  time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from
  manual_adjustments: true # false
  scaling_factor: 1.0

 costs:
  year: 2030
-  discountrate: 0.07 # From a Lion Hirth paper, also reflects average of Noothout et al 2016
-  USD2013_to_EUR2013: 0.7532 # [EUR/USD] ECB: https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html
-  marginal_cost: # EUR/MWh
+  version: v0.1.0
+  rooftop_share: 0.14  # based on the potentials, assuming  (0.1 kW/m2 and 10 m2/person)
+  fill_values:
+    FOM: 0
+    VOM: 0
+    efficiency: 1
+    fuel: 0
+    investment: 0
+    lifetime: 25
+    "CO2 intensity": 0
+    "discount rate": 0.07
+  marginal_cost:
    solar: 0.01
    onwind: 0.015
    offwind: 0.015
@ -251,6 +242,25 @@ costs:
  emission_prices: # in currency per tonne emission, only used with the option Ep
    co2: 0.

+clustering:
+  simplify_network:
+    to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections)
+    algorithm: kmeans # choose from: [hac, kmeans]
+    feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc.
+  cluster_network:
+    algorithm: kmeans
+    feature: solar+onwind-time
+  aggregation_strategies:
+    generators:
+      p_nom_max: sum # use "min" for more conservative assumptions
+      p_nom_min: sum
+      p_min_pu: mean
+      marginal_cost: mean
+      committable: any
+      ramp_limit_up: max
+      ramp_limit_down: max
+      efficiency: mean
+
 solving:
  options:
    formulation: kirchhoff
--- a/config.tutorial.yaml
+++ b/config.tutorial.yaml
@ -18,25 +18,6 @@ scenario:

 countries: ['BE']

-clustering:
-  simplify_network:
-    to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections)
-    algorithm: kmeans # choose from: [hac, kmeans]
-    feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc.
-  cluster_network:
-    algorithm: kmeans
-    feature: solar+onwind-time
-  aggregation_strategies:
-    generators:
-      p_nom_max: sum # use "min" for more conservative assumptions
-      p_nom_min: sum
-      p_min_pu: mean
-      marginal_cost: mean
-      committable: any
-      ramp_limit_up: max
-      ramp_limit_down: max
-      efficiency: mean
-
 snapshots:
  start: "2013-03-01"
  end: "2013-04-01"
@ -45,6 +26,7 @@ snapshots:
 enable:
  prepare_links_p_nom: false
  retrieve_databundle: true
+  retrieve_cost_data: true
  build_cutout: false
  retrieve_cutout: true
  build_natura_raster: false
@ -59,7 +41,7 @@ electricity:
    Generator: [OCGT]
    StorageUnit: [] #battery, H2
    Store: [battery, H2]
-    Link: []
+    Link: [] # H2 pipeline

  max_hours:
    battery: 6
@ -173,8 +155,17 @@ load:

 costs:
  year: 2030
-  discountrate: 0.07 # From a Lion Hirth paper, also reflects average of Noothout et al 2016
-  USD2013_to_EUR2013: 0.7532 # [EUR/USD] ECB: https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html
+  version: v0.1.0
+  rooftop_share: 0.14
+  fill_values:
+    FOM: 0
+    VOM: 0
+    efficiency: 1
+    fuel: 0
+    investment: 0
+    lifetime: 25
+    "CO2 intensity": 0
+    "discount rate": 0.07
  marginal_cost:
    solar: 0.01
    onwind: 0.015
@ -184,6 +175,25 @@ costs:
  emission_prices: # in currency per tonne emission, only used with the option Ep
    co2: 0.

+clustering:
+  simplify_network:
+    to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections)
+    algorithm: kmeans # choose from: [hac, kmeans]
+    feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc.
+  cluster_network:
+    algorithm: kmeans
+    feature: solar+onwind-time
+  aggregation_strategies:
+    generators:
+      p_nom_max: sum # use "min" for more conservative assumptions
+      p_nom_min: sum
+      p_min_pu: mean
+      marginal_cost: mean
+      committable: any
+      ramp_limit_up: max
+      ramp_limit_down: max
+      efficiency: mean
+
 solving:
  options:
    formulation: kirchhoff
--- a/doc/configtables/costs.csv
+++ b/doc/configtables/costs.csv
@ -1,8 +1,9 @@
 ,Unit,Values,Description
-year,--,"YYYY; e.g. '2030'","Year for which to retrieve cost assumptions of ``data/costs.csv``."
-discountrate,--,float,"Default discount rate if not specified for a technology in ``data/costs.csv``."
-USD2013_to_EUR2013,--,float,"Exchange rate from USD :math:`_{2013}` to EUR :math:`_{2013}` from `ECB <https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html>`_"
-capital_cost,EUR/MW,"Keys should be in the 'technology' column of ``data/costs.csv``. Values can be any float.","For the given technologies, assumptions about their capital investment costs are set to the corresponding value. Optional; overwrites cost assumptions from ``data/costs.csv``."
-marginal_cost,EUR/MWh,"Keys should be in the 'technology' column of ``data/costs.csv``. Values can be any float.","For the given technologies, assumptions about their marginal operating costs are set to the corresponding value. Optional; overwrites cost assumptions from ``data/costs.csv``."
+year,--,"YYYY; e.g. '2030'","Year for which to retrieve cost assumptions of ``resources/costs.csv``."
+version,--,"vX.X.X; e.g. 'v0.1.0'","Version of ``technology-data`` repository to use."
+rooftop_share,--,float,"Share of rooftop PV when calculating capital cost of solar (joint rooftop and utility-scale PV)."
+fill_values,--,float,"Default values if not specified for a technology in ``resources/costs.csv``."
+capital_cost,EUR/MW,"Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.","For the given technologies, assumptions about their capital investment costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``."
+marginal_cost,EUR/MWh,"Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.","For the given technologies, assumptions about their marginal operating costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``."
 emission_prices,,,"Specify exogenous prices for emission types listed in ``network.carriers`` to marginal costs."
 -- co2,EUR/t,float,"Exogenous price of carbon-dioxide added to the marginal costs of fossil-fuelled generators according to their carbon intensity. Added through the keyword ``Ep`` in the ``{opts}`` wildcard only in the rule :mod:`prepare_network``."
--- a/doc/configtables/snapshots.csv
+++ b/doc/configtables/snapshots.csv
@ -1,4 +1,4 @@
 ,Unit,Values,Description
 start,--,"str or datetime-like; e.g. YYYY-MM-DD","Left bound of date range"
 end,--,"str or datetime-like; e.g. YYYY-MM-DD","Right bound of date range"
-closed,--,"One of {None, ‘left’, ‘right’}","Make the time interval closed to the ``left``, ``right``, or both sides ``None``."
+closed,--,"One of {None, ‘left’, ‘right’}","Make the time interval closed to the ``left``, ``right``, or open on both sides ``None``."
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@ -265,8 +265,8 @@ Define additional generator attribute for conventional carrier types. If a scala
   :file: configtables/costs.csv

 .. note::
-    To change cost assumptions in more detail (i.e. other than ``marginal_cost`` and ``capital_cost``), consider modifying cost assumptions directly in ``data/costs.csv`` as this is not yet supported through the config file.
-    You can also build multiple different cost databases. Make a renamed copy of ``data/costs.csv`` (e.g. ``data/costs-optimistic.csv``) and set the variable ``COSTS=data/costs-optimistic.csv`` in the ``Snakefile``.
+    To change cost assumptions in more detail (i.e. other than ``marginal_cost`` and ``capital_cost``), consider modifying cost assumptions directly in ``resources/costs.csv`` as this is not yet supported through the config file.
+    You can also build multiple different cost databases. Make a renamed copy of ``resources/costs.csv`` (e.g. ``data/costs-optimistic.csv``) and set the variable ``COSTS=data/costs-optimistic.csv`` in the ``Snakefile``.

 .. _solving_cf:

--- a/doc/costs.rst
+++ b/doc/costs.rst
@ -7,7 +7,9 @@
 Cost Assumptions
 ##################

-The database of cost assumptions is stored in ``data/costs.csv``.
+The database of cost assumptions is retrieved from the repository `PyPSA/technology-data <https://github.com/pypsa/technology-data>`_ and then saved to``resources/costs.csv``. Cost assumptions of previous PyPSA-Eur versions can be restored by setting in the ``Snakefile``: ``COSTS="data/costs.csv".
+
+The ``config.yaml`` provides options to choose a reference year (``costs: year:``) and use a specific version of the repository ``costs: version:``.

 It includes cost assumptions for all included technologies for specific
 years from various sources, namely for
@ -39,15 +41,6 @@ Modifying Cost Assumptions

 Some cost assumptions (e.g. marginal cost and capital cost) can be directly overwritten in the ``config.yaml`` (cf. Section  :ref:`costs_cf`  in :ref:`config`).

-To change cost assumptions in more detail, modify cost assumptions directly in ``data/costs.csv`` as this is not yet supported through the config file.
+To change cost assumptions in more detail, modify cost assumptions directly in ``resources/costs.csv`` as this is not yet supported through the config file.

-You can also build multiple different cost databases. Make a renamed copy of ``data/costs.csv`` (e.g. ``data/costs-optimistic.csv``) and set the variable ``COSTS=data/costs-optimistic.csv`` in the ``Snakefile``.
-
-
-Default Cost Assumptions
-========================
-
-.. csv-table::
-   :header-rows: 1
-   :widths: 10,3,5,4,6,8
-   :file: ../data/costs.csv
+You can also build multiple different cost databases. Make a renamed copy of ``resources/costs.csv`` (e.g. ``data/costs-optimistic.csv``) and set the variable ``COSTS=data/costs-optimistic.csv`` in the ``Snakefile``.
--- a/doc/installation.rst
+++ b/doc/installation.rst
@ -102,6 +102,8 @@ It might be the case that you can only retrieve solutions by using a commercial
        conda activate pypsa-eur
        conda install -c conda-forge ipopt glpk

+.. warning::
+    On Windows, new versions of ``ipopt`` have caused problems. Consider downgrading to version 3.11.1.

 .. _defaultconfig:

--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@ -92,6 +92,11 @@ Upcoming Release

 * Hierarchical clustering was introduced. Distance metric is calculated from renewable potentials on hourly (feature entry ends with `-time`) or annual (feature entry in config end with `-cap`) values.

+* Techno-economic parameters of technologies (e.g. costs and efficiencies) will now be retrieved from a separate repository `PyPSA/technology-data <https://github.com/pypsa/technology-data>`_ 
+  that collects assumptions from a variety of sources. It is activated by default with ``enable: retrieve_cost_data: true`` and controlled with ``costs: year:`` and ``costs: version:``. 
+  The location of this data changed from ``data/costs.csv`` to ``resources/costs.csv`` 
+  [`#184 <https://github.com/PyPSA/pypsa-eur/pull/184>`_].
+

 Synchronisation Release - Ukraine and Moldova (17th March 2022)
 ===============================================================
@ -243,7 +248,6 @@ PyPSA-Eur 0.4.0 (22th September 2021)
  PyPSA network solving functions were not told about the solver logfile specified
  in the Snakemake file [`#247 <https://github.com/PyPSA/pypsa-eur/pull/247>`_]

-
 PyPSA-Eur 0.3.0 (7th December 2020)
 ===================================

--- a/doc/tutorial.rst
+++ b/doc/tutorial.rst
@ -47,7 +47,8 @@ The model can be adapted to only include selected countries (e.g. Belgium) inste

 .. literalinclude:: ../config.tutorial.yaml
   :language: yaml
-   :lines: 20
+   :start-at: countries:
+   :end-before: snapshots:

 Likewise, the example's temporal scope can be restricted (e.g. to a single month).

@ -60,14 +61,14 @@ It is also possible to allow less or more carbon-dioxide emissions. Here, we lim

 .. literalinclude:: ../config.tutorial.yaml
   :language: yaml
-   :lines: 40,42
+   :lines: 35,37

 PyPSA-Eur also includes a database of existing conventional powerplants.
-We can select which types of powerplants we like to be included with fixed capacities:
+We can select which types of powerplants we like to be included:

 .. literalinclude:: ../config.tutorial.yaml
   :language: yaml
-   :lines: 40,56
+   :lines: 35,51

 To accurately model the temporal and spatial availability of renewables such as wind and solar energy, we rely on historical weather data.
 It is advisable to adapt the required range of coordinates to the selection of countries.
@ -82,14 +83,14 @@ For example, we may want to use the ERA-5 dataset for solar and not the default

 .. literalinclude:: ../config.tutorial.yaml
   :language: yaml
-   :lines: 67,110,111
+   :lines: 62,105,106

 Finally, it is possible to pick a solver. For instance, this tutorial uses the open-source solvers CBC and Ipopt and does not rely
 on the commercial solvers Gurobi or CPLEX (for which free academic licenses are available).

 .. literalinclude:: ../config.tutorial.yaml
   :language: yaml
-   :lines: 173,183,184
+   :lines: 187,197,198

 .. note::

@ -215,7 +216,7 @@ A job (here ``simplify_network``) will display its attributes and normally some

    [<DATETIME>]
    rule simplify_network:
-        input: networks/elec.nc, data/costs.csv, resources/regions_onshore.geojson, resources/regions_offshore.geojson
+        input: networks/elec.nc, resources/costs.csv, resources/regions_onshore.geojson, resources/regions_offshore.geojson
        output: networks/elec_s.nc, resources/regions_onshore_elec_s.geojson, resources/regions_offshore_elec_s.geojson, resources/clustermaps_elec_s.h5
        jobid: 3
        benchmark: benchmarks/simplify_network/elec_s
@ -284,4 +285,4 @@ The solved networks can be analysed just like any other PyPSA network (e.g. in J

    network = pypsa.Network("results/networks/elec_s_6_ec_lcopt_Co2L-24H.nc")

-For inspiration, read the `examples section in the PyPSA documentation <https://pypsa.readthedocs.io/en/latest/examples.html>`_.
+For inspiration, read the `examples section in the PyPSA documentation <https://pypsa.readthedocs.io/en/latest/examples-basic.html>`_.
--- a/envs/environment.yaml
+++ b/envs/environment.yaml
@ -53,6 +53,8 @@ dependencies:
  - tqdm
  - pytz
  - tabula-py
+  - mergedeep
+  - pyxlsb

  - pip:
    - vresutils>=0.3.1
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@ -13,7 +13,7 @@ Relevant Settings

    costs:
        year:
-        USD2013_to_EUR2013:
+        version:
        dicountrate:
        emission_prices:

@ -46,14 +46,14 @@ Relevant Settings
 Inputs
 ------

- ``data/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity.
+- ``resources/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity.
 - ``data/bundle/hydro_capacities.csv``: Hydropower plant store/discharge power capacities, energy storage capacity, and average hourly inflow by country.

    .. image:: ../img/hydrocapacities.png
        :scale: 34 %

 - ``data/geth2015_hydro_capacities.csv``: alternative to capacities above; not currently used!
- ``resources/opsd_load.csv`` Hourly per-country load profiles.
+- ``resources/load.csv`` Hourly per-country load profiles.
 - ``resources/regions_onshore.geojson``: confer :ref:`busregions`
 - ``resources/nuts3_shapes.geojson``: confer :ref:`shapes`
 - ``resources/powerplants.csv``: confer :ref:`powerplants`
@ -93,7 +93,6 @@ import xarray as xr
 import geopandas as gpd
 import powerplantmatching as pm
 from powerplantmatching.export import map_country_bus
-
 from vresutils import transfer as vtransfer

 idx = pd.IndexSlice
@ -131,23 +130,14 @@ def _add_missing_carriers_from_costs(n, costs, carriers):
 def load_costs(tech_costs, config, elec_config, Nyears=1.):

    # set all asset costs and other parameters
-    costs = pd.read_csv(tech_costs, index_col=list(range(3))).sort_index()
+    costs = pd.read_csv(tech_costs, index_col=[0,1]).sort_index()

-    # correct units to MW and EUR
+    # correct units to MW
    costs.loc[costs.unit.str.contains("/kW"),"value"] *= 1e3
-    costs.loc[costs.unit.str.contains("USD"),"value"] *= config['USD2013_to_EUR2013']
+    costs.unit = costs.unit.str.replace("/kW", "/MW")

-    costs = (costs.loc[idx[:,config['year'],:], "value"]
-             .unstack(level=2).groupby("technology").sum(min_count=1))
-
-    costs = costs.fillna({"CO2 intensity" : 0,
-                          "FOM" : 0,
-                          "VOM" : 0,
-                          "discount rate" : config['discountrate'],
-                          "efficiency" : 1,
-                          "fuel" : 0,
-                          "investment" : 0,
-                          "lifetime" : 25})
+    fill_values = config["fill_values"]
+    costs = costs.value.unstack().fillna(fill_values)

    costs["capital_cost"] = ((calculate_annuity(costs["lifetime"], costs["discount rate"]) +
                             costs["FOM"]/100.) *
@ -163,8 +153,8 @@ def load_costs(tech_costs, config, elec_config, Nyears=1.):
    costs.at['OCGT', 'co2_emissions'] = costs.at['gas', 'co2_emissions']
    costs.at['CCGT', 'co2_emissions'] = costs.at['gas', 'co2_emissions']

-    costs.at['solar', 'capital_cost'] = 0.5*(costs.at['solar-rooftop', 'capital_cost'] +
-                                             costs.at['solar-utility', 'capital_cost'])
+    costs.at['solar', 'capital_cost'] = config["rooftop_share"] * costs.at['solar-rooftop', 'capital_cost'] + \
+                                        (1-config["rooftop_share"]) * costs.at['solar-utility', 'capital_cost']

    def costs_for_storage(store, link1, link2=None, max_hours=1.):
        capital_cost = link1['capital_cost'] + max_hours * store['capital_cost']
@ -179,7 +169,7 @@ def load_costs(tech_costs, config, elec_config, Nyears=1.):
        costs_for_storage(costs.loc["battery storage"], costs.loc["battery inverter"],
                          max_hours=max_hours['battery'])
    costs.loc["H2"] = \
-        costs_for_storage(costs.loc["hydrogen storage"], costs.loc["fuel cell"],
+        costs_for_storage(costs.loc["hydrogen storage underground"], costs.loc["fuel cell"],
                          costs.loc["electrolysis"], max_hours=max_hours['H2'])

    for attr in ('marginal_cost', 'capital_cost'):
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@ -13,7 +13,7 @@ Relevant Settings

    costs:
        year:
-        USD2013_to_EUR2013:
+        version:
        dicountrate:
        emission_prices:

@ -32,7 +32,7 @@ Relevant Settings
 Inputs
 ------

- ``data/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity.
+- ``resources/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity.

 Outputs
 -------
@ -76,16 +76,19 @@ def attach_storageunits(n, costs, elec_opts):
    lookup_dispatch = {"H2": "fuel cell", "battery": "battery inverter"}

    for carrier in carriers:
+        roundtrip_correction = 0.5 if carrier == "battery" else 1
+        
        n.madd("StorageUnit", buses_i, ' ' + carrier,
               bus=buses_i,
               carrier=carrier,
               p_nom_extendable=True,
               capital_cost=costs.at[carrier, 'capital_cost'],
               marginal_cost=costs.at[carrier, 'marginal_cost'],
-               efficiency_store=costs.at[lookup_store[carrier], 'efficiency'],
-               efficiency_dispatch=costs.at[lookup_dispatch[carrier], 'efficiency'],
+               efficiency_store=costs.at[lookup_store[carrier], 'efficiency']**roundtrip_correction,
+               efficiency_dispatch=costs.at[lookup_dispatch[carrier], 'efficiency']**roundtrip_correction,
               max_hours=max_hours[carrier],
-               cyclic_state_of_charge=True)
+               cyclic_state_of_charge=True
+        )


 def attach_stores(n, costs, elec_opts):
@ -104,7 +107,7 @@ def attach_stores(n, costs, elec_opts):
               carrier='H2',
               e_nom_extendable=True,
               e_cyclic=True,
-               capital_cost=costs.at["hydrogen storage", "capital_cost"])
+               capital_cost=costs.at["hydrogen storage underground", "capital_cost"])

        n.madd("Link", h2_buses_i + " Electrolysis",
               bus0=buses_i,
@ -140,7 +143,8 @@ def attach_stores(n, costs, elec_opts):
               bus0=buses_i,
               bus1=b_buses_i,
               carrier='battery charger',
-               efficiency=costs.at['battery inverter', 'efficiency'],
+               # the efficiencies are "round trip efficiencies"
+               efficiency=costs.at['battery inverter', 'efficiency']**0.5,
               capital_cost=costs.at['battery inverter', 'capital_cost'],
               p_nom_extendable=True,
               marginal_cost=costs.at["battery inverter", "marginal_cost"])
@ -149,7 +153,7 @@ def attach_stores(n, costs, elec_opts):
               bus0=b_buses_i,
               bus1=buses_i,
               carrier='battery discharger',
-               efficiency=costs.at['battery inverter','efficiency'],
+               efficiency=costs.at['battery inverter','efficiency']**0.5,
               p_nom_extendable=True,
               marginal_cost=costs.at["battery inverter", "marginal_cost"])

--- a/scripts/build_hydro_profile.py
+++ b/scripts/build_hydro_profile.py
@ -76,10 +76,32 @@ def get_eia_annual_hydro_generation(fn, countries):
    df = pd.read_csv(fn, skiprows=2, index_col=1, na_values=[u' ','--']).iloc[1:, 1:]
    df.index = df.index.str.strip()

+    former_countries = {
+        "Former Czechoslovakia": dict(
+            countries=["Czech Republic", "Slovakia"],
+            start=1980, end=1992),
+        "Former Serbia and Montenegro": dict(
+            countries=["Serbia", "Montenegro"],
+            start=1992, end=2005),
+        "Former Yugoslavia": dict(
+            countries=["Slovenia", "Croatia", "Bosnia and Herzegovina", "Serbia", "Montenegro", "North Macedonia"],
+            start=1980, end=1991),
+    }
+
+    for k, v in former_countries.items():
+        period = [str(i) for i in range(v["start"], v["end"]+1)]
+        ratio = df.loc[v['countries']].T.dropna().sum()
+        ratio /= ratio.sum()
+        for country in v['countries']:
+            df.loc[country, period] = df.loc[k, period] * ratio[country]
+
+    baltic_states = ["Latvia", "Estonia", "Lithuania"]
+    df.loc[baltic_states] = df.loc[baltic_states].T.fillna(df.loc[baltic_states].mean(axis=1)).T
+
    df.loc["Germany"] = df.filter(like='Germany', axis=0).sum()
-    df.loc["Serbia"] += df.loc["Kosovo"]
+    df.loc["Serbia"] += df.loc["Kosovo"].fillna(0.)
    df = df.loc[~df.index.str.contains('Former')]
-    df.drop(["Europe", "Germany, West", "Germany, East"], inplace=True)
+    df.drop(["Europe", "Germany, West", "Germany, East", "Kosovo"], inplace=True)

    df.index = cc.convert(df.index, to='iso2')
    df.index.name = 'countries'
--- a/scripts/build_load_data.py
+++ b/scripts/build_load_data.py
@ -26,11 +26,12 @@ Relevant Settings
 Inputs
 ------

+- ``data/load_raw.csv``:

 Outputs
 -------

- ``resource/time_series_60min_singleindex_filtered.csv``:
+- ``resources/load.csv``:


 """
--- a/scripts/cluster_network.py
+++ b/scripts/cluster_network.py
@ -122,7 +122,7 @@ Exemplary unsolved network clustered to 37 nodes:
 """

 import logging
-from _helpers import configure_logging, update_p_nom_max, get_aggregation_strategies, REGION_COLS
+from _helpers import configure_logging, update_p_nom_max, get_aggregation_strategies

 import pypsa
 import os
@ -372,9 +372,8 @@ def cluster_regions(busmaps, input=None, output=None):

    for which in ('regions_onshore', 'regions_offshore'):
        regions = gpd.read_file(getattr(input, which))
-        regions = regions.reindex(columns=REGION_COLS).set_index('name')
-        aggfunc = dict(x="mean", y="mean", country="first")
-        regions_c = regions.dissolve(busmap, aggfunc=aggfunc)
+        regions = regions.reindex(columns=["name", "geometry"]).set_index('name')
+        regions_c = regions.dissolve(busmap)
        regions_c.index.name = 'name'
        regions_c = regions_c.reset_index()
        regions_c.to_file(getattr(output, which))
--- a/scripts/make_summary.py
+++ b/scripts/make_summary.py
@ -11,8 +11,9 @@ Relevant Settings
 .. code:: yaml

    costs:
-        USD2013_to_EUR2013:
-        discountrate:
+        year:
+        version:
+        fill_values:
        marginal_cost:
        capital_cost:

--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@ -20,9 +20,10 @@ Relevant Settings
 .. code:: yaml

    costs:
+        year:
+        version:
+        fill_values:
        emission_prices:
-        USD2013_to_EUR2013:
-        discountrate:
        marginal_cost:
        capital_cost:

@ -37,7 +38,7 @@ Relevant Settings
 Inputs
 ------

- ``data/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity.
+- ``resources/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity.
 - ``networks/elec_s{simpl}_{clusters}.nc``: confer :ref:`cluster`

 Outputs
--- a/scripts/retrieve_databundle.py
+++ b/scripts/retrieve_databundle.py
@ -11,7 +11,7 @@ The data bundle (1.4 GB) contains common GIS datasets like NUTS3 shapes, EEZ sha

 This rule downloads the data bundle from `zenodo <https://doi.org/10.5281/zenodo.3517935>`_ and extracts it in the ``data`` sub-directory, such that all files of the bundle are stored in the ``data/bundle`` subdirectory.

-The :ref:`tutorial` uses a smaller `data bundle <https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz>`_ than required for the full model (19 MB)
+The :ref:`tutorial` uses a smaller `data bundle <https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz>`_ than required for the full model (188 MB)

 .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517921.svg
    :target: https://doi.org/10.5281/zenodo.3517921
@ -28,7 +28,7 @@ The :ref:`tutorial` uses a smaller `data bundle <https://zenodo.org/record/35179

 **Outputs**

- ``cutouts/bundle``: input data collected from various sources
+- ``data/bundle``: input data collected from various sources

 """

--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@ -19,8 +19,9 @@ Relevant Settings
      aggregation_strategies:

    costs:
-        USD2013_to_EUR2013:
-        discountrate:
+        year:
+        version:
+        fill_values:
        marginal_cost:
        capital_cost:

@ -45,7 +46,7 @@ Relevant Settings
 Inputs
 ------

- ``data/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity.
+- ``resources/costs.csv``: The database of cost assumptions for all included technologies for specific years from various sources; e.g. discount rate, lifetime, investment (CAPEX), fixed operation and maintenance (FOM), variable operation and maintenance (VOM), fuel costs, efficiency, carbon-dioxide intensity.
 - ``resources/regions_onshore.geojson``: confer :ref:`busregions`
 - ``resources/regions_offshore.geojson``: confer :ref:`busregions`
 - ``networks/elec.nc``: confer :ref:`electricity`
--- a/test/config.test1.yaml
+++ b/test/config.test1.yaml
@ -17,25 +17,6 @@ scenario:

 countries: ['BE']

-clustering:
-  simplify_network:
-    to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections)
-    algorithm: kmeans # choose from: [hac, kmeans]
-    feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc.
-  cluster_network:
-    algorithm: kmeans
-    feature: solar+onwind-time
-  aggregation_strategies:
-    generators:
-      p_nom_max: sum # use "min" for more conservative assumptions
-      p_nom_min: sum
-      p_min_pu: mean
-      marginal_cost: mean
-      committable: any
-      ramp_limit_up: max
-      ramp_limit_down: max
-      efficiency: mean
-
 snapshots:
  start: "2013-03-01"
  end: "2013-03-08"
@ -44,6 +25,7 @@ snapshots:
 enable:
  prepare_links_p_nom: false
  retrieve_databundle: true
+  retrieve_cost_data: true
  build_cutout: false
  retrieve_cutout: true
  build_natura_raster: false
@ -171,8 +153,17 @@ load:

 costs:
  year: 2030
-  discountrate: 0.07 # From a Lion Hirth paper, also reflects average of Noothout et al 2016
-  USD2013_to_EUR2013: 0.7532 # [EUR/USD] ECB: https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html
+  version: v0.1.0
+  rooftop_share: 0.14
+  fill_values:
+    FOM: 0
+    VOM: 0
+    efficiency: 1
+    fuel: 0
+    investment: 0
+    lifetime: 25
+    "CO2 intensity": 0
+    "discount rate": 0.07
  marginal_cost:
    solar: 0.01
    onwind: 0.015
@ -182,6 +173,25 @@ costs:
  emission_prices: # only used with the option Ep
    co2: 0.

+clustering:
+  simplify_network:
+    to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections)
+    algorithm: kmeans # choose from: [hac, kmeans]
+    feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc.
+  cluster_network:
+    algorithm: kmeans
+    feature: solar+onwind-time
+  aggregation_strategies:
+    generators:
+      p_nom_max: sum # use "min" for more conservative assumptions
+      p_nom_min: sum
+      p_min_pu: mean
+      marginal_cost: mean
+      committable: any
+      ramp_limit_up: max
+      ramp_limit_down: max
+      efficiency: mean
+
 solving:
  options:
    formulation: kirchhoff