Merge branch 'master' into retrofit-gas-pipelines

Snakefile

@@ -1,7 +1,13 @@
 
+from os.path import exists
+from shutil import copyfile
+
 from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider
 HTTP = HTTPRemoteProvider()
 
+if not exists("config.yaml"):
+    copyfile("config.default.yaml", "config.yaml")
+
 configfile: "config.yaml"
 
 
@@ -38,6 +44,22 @@ rule prepare_sector_networks:
         expand(RDIR + "/prenetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc",
                **config['scenario'])
 
+datafiles = [
+    "eea/UNFCCC_v23.csv",
+    "switzerland-sfoe/switzerland-new_format.csv",
+    "nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson",
+    "myb1-2017-nitro.xls",
+    "Industrial_Database.csv",
+    "emobility/KFZ__count",
+    "emobility/Pkw__count",
+]
+
+if config.get('retrieve_sector_databundle', True):
+    rule retrieve_sector_databundle:
+        output:  expand('data/{file}', file=datafiles)
+        log: "logs/retrieve_sector_databundle.log"
+        script: 'scripts/retrieve_sector_databundle.py'
+
 
 rule build_population_layouts:
     input:
@@ -258,6 +280,19 @@ else:
     build_biomass_transport_costs_output = {}
 
 
+rule build_salt_cavern_potentials:
+    input:
+        salt_caverns="data/h2_salt_caverns_GWh_per_sqkm.geojson",
+        regions_onshore=pypsaeur("resources/regions_onshore_elec_s{simpl}_{clusters}.geojson"),
+        regions_offshore=pypsaeur("resources/regions_offshore_elec_s{simpl}_{clusters}.geojson"),
+    output:
+        h2_cavern_potential="resources/salt_cavern_potentials_s{simpl}_{clusters}.csv"
+    threads: 1
+    resources: mem_mb=2000
+    benchmark: "benchmarks/build_salt_cavern_potentials_s{simpl}_{clusters}"
+    script: "scripts/build_salt_cavern_potentials.py"
+
+
 rule build_ammonia_production:
     input:
         usgs="data/myb1-2017-nitro.xls"
@@ -406,7 +441,7 @@ rule prepare_sector_network:
         costs=CDIR + "costs_{planning_horizons}.csv",
         profile_offwind_ac=pypsaeur("resources/profile_offwind-ac.nc"),
         profile_offwind_dc=pypsaeur("resources/profile_offwind-dc.nc"),
-        h2_cavern="data/hydrogen_salt_cavern_potentials.csv",
+        h2_cavern="resources/salt_cavern_potentials_s{simpl}_{clusters}.csv",
         busmap_s=pypsaeur("resources/busmap_elec_s{simpl}.csv"),
         busmap=pypsaeur("resources/busmap_elec_s{simpl}_{clusters}.csv"),
         clustered_pop_layout="resources/pop_layout_elec_s{simpl}_{clusters}.csv",
@@ -524,7 +559,7 @@ if config["foresight"] == "overnight":
             solver=RDIR + "/logs/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_solver.log",
             python=RDIR + "/logs/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_python.log",
             memory=RDIR + "/logs/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}_memory.log"
-        threads: 4
+        threads: config['solving']['solver'].get('threads', 4)
         resources: mem_mb=config['solving']['mem']
         benchmark: RDIR + "/benchmarks/solve_network/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}"
         script: "scripts/solve_network.py"

config.default.yaml

@@ -2,6 +2,8 @@ version: 0.6.0
 
 logging_level: INFO
 
+retrieve_sector_databundle: true
+
 results_dir: results/
 summary_dir: results
 costs_dir: ../technology-data/outputs/
@@ -238,6 +240,10 @@ sector:
   co2_network: false
   cc_fraction: 0.9  # default fraction of CO2 captured with post-combustion capture
   hydrogen_underground_storage: true
+  hydrogen_underground_storage_locations:
+    - onshore  # more than 50 km from sea
+    # - nearshore  # within 50 km of sea
+    # - offshore
   use_fischer_tropsch_waste_heat: true
   use_fuel_cell_waste_heat: true
   electricity_distribution_grid: false

doc/data.csv

@@ -15,7 +15,7 @@ co2 budgets,co2_budget.csv,CC BY 4.0,https://arxiv.org/abs/2004.11009
 existing heating potentials,existing_infrastructure/existing_heating_raw.csv,unknown,https://ec.europa.eu/energy/studies/mapping-and-analyses-current-and-future-2020-2030-heatingcooling-fuel-deployment_en?redir=1
 IRENA existing VRE capacities,existing_infrastructure/{solar|onwind|offwind}_capcity_IRENA.csv,unknown,https://www.irena.org/Statistics/Download-Data
 USGS ammonia production,myb1-2017-nitro.xls,unknown,https://www.usgs.gov/centers/nmic/nitrogen-statistics-and-information
-hydrogen salt cavern potentials,hydrogen_salt_cavern_potentials.csv,CC BY 4.0,https://doi.org/10.1016/j.ijhydene.2019.12.161
+hydrogen salt cavern potentials,h2_salt_caverns_GWh_per_sqkm.geojson,CC BY 4.0,https://doi.org/10.1016/j.ijhydene.2019.12.161 https://doi.org/10.20944/preprints201910.0187.v1
 hotmaps industrial site database,Industrial_Database.csv,CC BY 4.0,https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database
 Hotmaps building stock data,data_building_stock.csv,CC BY 4.0,https://gitlab.com/hotmaps/building-stock
 U-values Poland,u_values_poland.csv,unknown,https://data.europa.eu/euodp/de/data/dataset/building-stock-observatory

doc/installation.rst

@@ -56,9 +56,11 @@ atlite version 0.0.2.
 
 You can create an enviroment using the environment.yaml file in pypsa-eur/envs:
 
-.../pypsa-eur % conda env create -f envs/environment.yaml
+.. code:: bash
 
-.../pypsa-eur % conda activate pypsa-eur
+    .../pypsa-eur % conda env create -f envs/environment.yaml
+
+    .../pypsa-eur % conda activate pypsa-eur
 
 See details in `PyPSA-Eur Installation <https://pypsa-eur.readthedocs.io/en/latest/installation.html>`_
 
@@ -72,9 +74,9 @@ The data bundle's size is around 640 MB.
 To download and extract the data bundle on the command line:
 
 .. code:: bash
-`
+
     projects/pypsa-eur-sec/data % wget "https://zenodo.org/record/5546517/files/pypsa-eur-sec-data-bundle.tar.gz"
-    projects/pypsa-eur-sec/data % tar xvzf pypsa-eur-sec-data-bundle.tar.gz
+    projects/pypsa-eur-sec/data % tar -xvzf pypsa-eur-sec-data-bundle.tar.gz
 
 
 The data licences and sources are given in the following table.

doc/release_notes.rst

@@ -53,6 +53,10 @@ incorporates retrofitting options to hydrogen.
   transmission routes. Previously, only the electricity transmission routes were
   considered.
 
+* Option ``retrieve_sector_databundle`` to automatically retrieve and extract data bundle.
+
+* Add regionalised hydrogen salt cavern storage potentials from `Technical Potential of Salt Caverns for Hydrogen Storage in Europe <https://doi.org/10.20944/preprints201910.0187.v1>`_.
+
 
 PyPSA-Eur-Sec 0.6.0 (4 October 2021)
 ====================================

scripts/build_industry_sector_ratios.py

@@ -1006,7 +1006,7 @@ def non_ferrous_metals():
     # Alumina
 
     # High enthalpy heat is converted to methane.
-    # Process heat at T>500ºC is required here.
+    # Process heat at T>500C is required here.
     # Refining is electrified.
     # There are no process emissions associated to Alumina manufacturing.
 

scripts/build_salt_cavern_potentials.py

@@ -0,0 +1,78 @@
+"""
+Build salt cavern potentials for hydrogen storage.
+
+Technical Potential of Salt Caverns for Hydrogen Storage in Europe
+CC-BY 4.0
+https://doi.org/10.20944/preprints201910.0187.v1
+https://doi.org/10.1016/j.ijhydene.2019.12.161
+
+Figure 6. Distribution of potential salt cavern sites across Europe with their corresponding
+energy densities (cavern storage potential divided by the volume).
+
+Figure 7. Total cavern storage potential in European countries
+classified as onshore, offshore and within 50 km of shore.
+
+The regional distribution is taken from the map (Figure 6) and scaled to the
+capacities from the bar chart split by nearshore (<50km from sea),
+onshore (>50km from sea), offshore (Figure 7).
+"""
+
+
+import geopandas as gpd
+import pandas as pd
+
+
+def concat_gdf(gdf_list, crs='EPSG:4326'):
+    """Concatenate multiple geopandas dataframes with common coordinate reference system (crs)."""
+    return gpd.GeoDataFrame(pd.concat(gdf_list), crs=crs)
+
+
+def load_bus_regions(onshore_path, offshore_path):
+    """Load pypsa-eur on- and offshore regions and concat."""
+
+    bus_regions_offshore = gpd.read_file(offshore_path)
+    bus_regions_onshore = gpd.read_file(onshore_path)
+    bus_regions = concat_gdf([bus_regions_offshore, bus_regions_onshore])
+    bus_regions = bus_regions.dissolve(by='name', aggfunc='sum')
+
+    return bus_regions
+
+
+def area(gdf):
+    """Returns area of GeoDataFrame geometries in square kilometers."""
+    return gdf.to_crs(epsg=3035).area.div(1e6)
+
+
+def salt_cavern_potential_by_region(caverns, regions):
+
+    # calculate area of caverns shapes
+    caverns["area_caverns"] = area(caverns)
+
+    overlay = gpd.overlay(regions.reset_index(), caverns, keep_geom_type=True)
+
+    # calculate share of cavern area inside region
+    overlay["share"] = area(overlay) / overlay["area_caverns"]
+
+    overlay["e_nom"] = overlay.eval("capacity_per_area * share * area_caverns / 1000") # TWh
+
+    caverns_regions = overlay.groupby(['name', "storage_type"]).e_nom.sum().unstack("storage_type")
+    
+    return caverns_regions
+
+
+if __name__ == '__main__':
+    if 'snakemake' not in globals():
+        from helper import mock_snakemake
+        snakemake = mock_snakemake('build_salt_cavern_potentials', simpl='', clusters='37')
+
+
+    fn_onshore = snakemake.input.regions_onshore
+    fn_offshore = snakemake.input.regions_offshore
+
+    regions = load_bus_regions(fn_onshore, fn_offshore)
+
+    caverns = gpd.read_file(snakemake.input.salt_caverns)  # GWh/sqkm
+
+    caverns_regions = salt_cavern_potential_by_region(caverns, regions)
+
+    caverns_regions.to_csv(snakemake.output.h2_cavern_potential)

scripts/copy_config.py

@@ -1,18 +1,18 @@
 
 from shutil import copy
 
-files = [
+files = {
-    "config.yaml",
+    "config.yaml": "config.yaml",
-    "Snakefile",
+    "Snakefile": "Snakefile",
-    "scripts/solve_network.py",
+    "scripts/solve_network.py": "solve_network.py",
-    "scripts/prepare_sector_network.py",
+    "scripts/prepare_sector_network.py": "prepare_sector_network.py",
-    "../pypsa-eur/config.yaml"
+    "../pypsa-eur/config.yaml": "config.pypsaeur.yaml"
-]
+}
 
 if __name__ == '__main__':
     if 'snakemake' not in globals():
         from helper import mock_snakemake
         snakemake = mock_snakemake('copy_config')
 
-    for f in files:
+    for f, name in files.items():
-        copy(f,snakemake.config['summary_dir'] + '/' + snakemake.config['run'] + '/configs/')
+        copy(f,snakemake.config['summary_dir'] + '/' + snakemake.config['run'] + '/configs/' + name)

scripts/prepare_sector_network.py

@@ -1039,26 +1039,27 @@ def add_storage_and_grids(n, costs):
         lifetime=costs.at['fuel cell', 'lifetime']
     )
 
-    cavern_nodes = pd.DataFrame()
+    cavern_types = snakemake.config["sector"]["hydrogen_underground_storage_locations"]
+    h2_caverns = pd.read_csv(snakemake.input.h2_cavern, index_col=0)[cavern_types].sum(axis=1)
+
+    # only use sites with at least 2 TWh potential
+    h2_caverns = h2_caverns[h2_caverns > 2]
+    
+    # convert TWh to MWh
+    h2_caverns = h2_caverns * 1e6
+
+    # clip at 1000 TWh for one location
+    h2_caverns.clip(upper=1e9, inplace=True)
+
     if options['hydrogen_underground_storage']:
-         h2_salt_cavern_potential = pd.read_csv(snakemake.input.h2_cavern, index_col=0, squeeze=True)
-         h2_cavern_ct = h2_salt_cavern_potential[~h2_salt_cavern_potential.isna()]
-         cavern_nodes = pop_layout[pop_layout.ct.isin(h2_cavern_ct.index)]
 
-         h2_capital_cost = costs.at["hydrogen storage underground", "fixed"]
+        h2_capital_cost = costs.at["hydrogen storage underground", "fixed"]
 
-         # assumptions: weight storage potential in a country by population
+        n.madd("Store",
-         # TODO: fix with real geographic potentials
+            h2_caverns.index + " H2 Store",
-         # convert TWh to MWh with 1e6
+            bus=h2_caverns.index + " H2",
-         h2_pot = h2_cavern_ct.loc[cavern_nodes.ct]
-         h2_pot.index = cavern_nodes.index
-         h2_pot = h2_pot * cavern_nodes.fraction * 1e6
-
-         n.madd("Store",
-            cavern_nodes.index + " H2 Store",
-            bus=cavern_nodes.index + " H2",
             e_nom_extendable=True,
-            e_nom_max=h2_pot.values,
+            e_nom_max=h2_caverns.values,
             e_cyclic=True,
             carrier="H2 Store",
             capital_cost=h2_capital_cost
@@ -1066,7 +1067,7 @@ def add_storage_and_grids(n, costs):
 
     # hydrogen stored overground (where not already underground)
     h2_capital_cost = costs.at["hydrogen storage tank incl. compressor", "fixed"]
-    nodes_overground = cavern_nodes.index.symmetric_difference(nodes)
+    nodes_overground = h2_caverns.index.symmetric_difference(nodes)
 
     n.madd("Store",
         nodes_overground + " H2 Store",

scripts/retrieve_sector_databundle.py

@@ -0,0 +1,35 @@
+"""
+Retrieve and extract sector data bundle.
+"""
+
+import logging
+logger = logging.getLogger(__name__)
+
+import os
+import sys
+import tarfile
+from pathlib import Path
+
+# Add pypsa-eur scripts to path for import of _helpers
+sys.path.insert(0, os.getcwd() + "/../pypsa-eur/scripts")
+
+from _helpers import progress_retrieve, configure_logging
+
+
+if __name__ == "__main__":
+    configure_logging(snakemake)
+
+    url = "https://zenodo.org/record/5546517/files/pypsa-eur-sec-data-bundle.tar.gz"
+
+    tarball_fn = Path("sector-bundle.tar.gz")
+    to_fn = Path("data")
+
+    logger.info(f"Downloading databundle from '{url}'.")
+    progress_retrieve(url, tarball_fn)
+
+    logger.info(f"Extracting databundle.")
+    tarfile.open(tarball_fn).extractall(to_fn)
+
+    tarball_fn.unlink()
+
+    logger.info(f"Databundle available in '{to_fn}'.")