add option to automatically download and extract sector databundle (#198)

Snakefile

@@ -44,6 +44,22 @@ rule prepare_sector_networks:
         expand(RDIR + "/prenetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_{planning_horizons}.nc",
                **config['scenario'])
 
+datafiles = [
+    "eea/UNFCCC_v23.csv",
+    "switzerland-sfoe/switzerland-new_format.csv",
+    "nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson",
+    "myb1-2017-nitro.xls",
+    "Industrial_Database.csv",
+    "emobility/KFZ__count",
+    "emobility/Pkw__count",
+]
+
+if config.get('retrieve_sector_databundle', True):
+    rule retrieve_sector_databundle:
+        output:  expand('data/{file}', file=datafiles)
+        log: "logs/retrieve_sector_databundle.log"
+        script: 'scripts/retrieve_sector_databundle.py'
+
 
 rule build_population_layouts:
     input:

config.default.yaml

@@ -2,6 +2,8 @@ version: 0.6.0
 
 logging_level: INFO
 
+retrieve_sector_databundle: true
+
 results_dir: results/
 summary_dir: results
 costs_dir: ../technology-data/outputs/

doc/release_notes.rst

@@ -8,8 +8,9 @@ Future release
 .. note::
   This unreleased version currently may require the master branches of PyPSA, PyPSA-Eur, and the technology-data repository.
 
-* Add regionalised hydrogen salt cavern storage potentials from `Technical Potential of Salt Caverns for Hydrogen Storage in Europe <https://doi.org/10.20944/preprints201910.0187.v1>`_.
+* Option ``retrieve_sector_databundle`` to automatically retrieve and extract data bundle.
 
+* Add regionalised hydrogen salt cavern storage potentials from `Technical Potential of Salt Caverns for Hydrogen Storage in Europe <https://doi.org/10.20944/preprints201910.0187.v1>`_.
 
 PyPSA-Eur-Sec 0.6.0 (4 October 2021)
 ====================================

scripts/retrieve_sector_databundle.py

@@ -0,0 +1,35 @@
+"""
+Retrieve and extract sector data bundle.
+"""
+
+import logging
+logger = logging.getLogger(__name__)
+
+import os
+import sys
+import tarfile
+from pathlib import Path
+
+# Add pypsa-eur scripts to path for import of _helpers
+sys.path.insert(0, os.getcwd() + "/../pypsa-eur/scripts")
+
+from _helpers import progress_retrieve, configure_logging
+
+
+if __name__ == "__main__":
+    configure_logging(snakemake)
+
+    url = "https://zenodo.org/record/5546517/files/pypsa-eur-sec-data-bundle.tar.gz"
+
+    tarball_fn = Path("sector-bundle.tar.gz")
+    to_fn = Path("data")
+
+    logger.info(f"Downloading databundle from '{url}'.")
+    progress_retrieve(url, tarball_fn)
+
+    logger.info(f"Extracting databundle.")
+    tarfile.open(tarball_fn).extractall(to_fn)
+
+    tarball_fn.unlink()
+
+    logger.info(f"Databundle available in '{to_fn}'.")