diff --git a/doc/configtables/toplevel.csv b/doc/configtables/toplevel.csv index dfc4a0bb..11713eb2 100644 --- a/doc/configtables/toplevel.csv +++ b/doc/configtables/toplevel.csv @@ -1,8 +1,10 @@ ,Unit,Values,Description version,--,0.1,"Version of PyPSA-Eur" +tutorial,bool,"{true, false}","Switch to retrieve the tutorial data set instead of the full data set." logging_level,--,"Any of {'INFO', 'WARNING', 'ERROR'}","Restrict console outputs to all infos, warning or errors only" summary_dir,--,"e.g. 'results'","Directory into which results are written." countries,--,"Subset of {'AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'}","European countries defined by their `Two-letter country codes (ISO 3166-1) `_ which should be included in the energy system model." enable,,, --- powerplantmatching,bool,"{true, false}","Switch to retrieve currently existing conventional power plant capacities matched from multiple sources by using `powerplantmatching `_." --- prepare_links_p_nom,bool,"{true, false}","Switch to retrieve current HVDC projects from `Wikipedia `_" \ No newline at end of file +-- prepare_links_p_nom,bool,"{true, false}","Switch to retrieve current HVDC projects from `Wikipedia `_" +-- build_cutout,bool,"{true, false}","Switch to build cutouts via the rule :mod:`build_cutout` or whether to retrieve the data from zenodo with :mod:`retrieve_cutout`." +-- build_natura_raster,bool,"{true, false}","Switch to create raster ``natura.tiff`` via the rule :mod:`build_natura_raster` or whether to retrieve ``natura.tiff`` from zenodo with :mod:`retrieve_natura_raster`." \ No newline at end of file diff --git a/doc/installation.rst b/doc/installation.rst index 33dde1d4..66609e04 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -43,70 +43,7 @@ The environment can be installed and activated using .. note:: Note that activation is local to the currently open shell! - After opening a new terminal window, one needs to reissue the second command! - -.. _data: - -Download Data Dependencies -============================== - -Not all data dependencies are shipped with the git repository, -since git is not suited for handling large changing files. -Instead we provide separate data bundles which can be obtained -using the described shell commands or by downloading and -extracting them manually in the locations outlined below. - -.. note:: - - The :ref:`tutorial` uses smaller data bundles than required for the full model. - To start with the tutorial, substitute with the links below using the following alternatives: - - .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517921.svg - :target: https://doi.org/10.5281/zenodo.3517921 - - .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3518020.svg - :target: https://doi.org/10.5281/zenodo.3518020 - - - **Data Bundle:** ``https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz`` (197 MB) - - **Cutouts:** ``https://zenodo.org/record/3518020/files/pypsa-eur-tutorial-cutouts.tar.xz`` (19 MB) - -.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517935.svg - :target: https://doi.org/10.5281/zenodo.3517935 - -1. **Data Bundle:** `pypsa-eur-data-bundle.tar.xz `_ (1.4 GB) contains common GIS datasets like NUTS3 shapes, EEZ shapes, CORINE Landcover, Natura 2000 and also electricity specific summary statistics like historic per country yearly totals of hydro generation, GDP and POP on NUTS3 levels and per-country load time-series. It should be extracted in the ``data`` sub-directory, such that all files of the bundle are stored in the ``data/bundle`` subdirectory) - -.. code:: bash - - .../pypsa-eur/data % curl -OL "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz" - - .../pypsa-eur/data % tar xJf pypsa-eur-data-bundle.tar.xz - -.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517949.svg - :target: https://doi.org/10.5281/zenodo.3517949 - -2. **Cutouts:** `pypsa-eur-cutouts.tar.xz `_ (3.9 GB) are spatiotemporal subsets of the European weather data from the `ECMWF ERA5 `_ reanalysis dataset and the `CMSAF SARAH-2 `_ solar surface radiation dataset for the year 2013. They have been prepared by and are for use with the `atlite `_ tool. You can either generate them yourself using the ``build_cutouts`` rule or extract them directly into the ``pypsa-eur`` directory. To download cutouts yourself you need to `set up the CDS API `_. For more details read the `atlite documentation `_. For beginners, extracting the bundle is recommended: - -.. code:: bash - - .../pypsa-eur % curl -OL "https://zenodo.org/record/3517949/files/pypsa-eur-cutouts.tar.xz" - - .../pypsa-eur % tar xJf pypsa-eur-cutouts.tar.xz - -.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3518215.svg - :target: https://doi.org/10.5281/zenodo.3518215 - -3. **Natura:** Optionally, you can download a rasterized version of the NATURA dataset `natura.tiff `_ and put it into the ``resources`` sub-directory. If you don't, it will be generated automatically, which is a time-consuming process. - -.. code:: bash - - .../pypsa-eur % curl -L "https://zenodo.org/record/3518215/files/natura.tiff" -o "resources/natura.tiff" - - -4. **Remove Archives:** Optionally, if you want to save disk space, you can delete ``data/pypsa-eur-data-bundle.tar.xz`` and ``pypsa-eur-cutouts.tar.xz`` once extracting the bundles is complete. E.g. - -.. code:: bash - - .../pypsa-eur % rm -rf data/pypsa-eur-data-bundle.tar.xz pypsa-eur-cutouts.tar.xz + After opening a new terminal window, one needs to reissue the second command! Install a Solver ================ diff --git a/doc/preparation.rst b/doc/preparation.rst index 5fb891dc..8694c1f7 100644 --- a/doc/preparation.rst +++ b/doc/preparation.rst @@ -3,7 +3,12 @@ Preparing Networks ########################################## The preparation process of the PyPSA-Eur energy system model consists of a group of ``snakemake`` -rules which are briefly outlined and explained in detail in the sections below: +rules which are briefly outlined and explained in detail in the sections below. + +Not all data dependencies are shipped with the git repository. +Instead we provide separate data bundles which can be obtained +using the ``retrieve*`` rules (:ref:`data`). +Having downloaded the necessary data, - :mod:`build_shapes` generates GeoJSON files with shapes of the countries, exclusive economic zones and `NUTS3 `_ areas. - :mod:`build_cutout` prepares smaller weather data portions from `ERA5 `_ for cutout ``europe-2013-era5`` and SARAH for cutout ``europe-2013-sarah``. @@ -17,6 +22,7 @@ With these and the externally extracted ENTSO-E online map topology Then the process continues by calculating conventional power plant capacities, potentials, and per-unit availability time series for variable renewable energy carriers and hydro power plants with the following rules: - :mod:`build_powerplants` for today's thermal power plant capacities using `powerplantmatching `_ allocating these to the closest substation for each powerplant, +- :mod:`build_natura_raster` for rasterising NATURA2000 natural protection areas, - :mod:`build_renewable_profiles` for the hourly capacity factors and installation potentials constrained by land-use in each substation's Voronoi cell for PV, onshore and offshore wind, and - :mod:`build_hydro_profile` for the hourly per-unit hydro power availability time series. @@ -26,8 +32,10 @@ together into a detailed PyPSA network stored in ``networks/elec.nc``. .. toctree:: :caption: Overview + preparation/retrieve preparation/build_shapes preparation/build_cutout + preparation/build_natura_raster preparation/prepare_links_p_nom preparation/base_network preparation/build_bus_regions diff --git a/doc/preparation/retrieve.rst b/doc/preparation/retrieve.rst new file mode 100644 index 00000000..521dfa57 --- /dev/null +++ b/doc/preparation/retrieve.rst @@ -0,0 +1,24 @@ +.. _data: + +Rules ``retrieve*`` +============================= + +Not all data dependencies are shipped with the git repository, +since git is not suited for handling large changing files. +Instead we provide separate data bundles which can be obtained +using the ``retrieve*`` rules. + +Rule ``retrieve_databundle`` +---------------------------- + +.. automodule:: retrieve_databundle + +Rule ``retrieve_cutout`` +------------------------ + +.. automodule:: retrieve_cutout + +Rule ``retrieve_natura_raster`` +------------------------------- + +.. automodule:: retrieve_natura_raster diff --git a/doc/tutorial.rst b/doc/tutorial.rst index 57dd5bb2..1ca89d3e 100644 --- a/doc/tutorial.rst +++ b/doc/tutorial.rst @@ -12,25 +12,13 @@ which allows the user to explore most of its functionalities on a local machine. It takes approximately five minutes to complete and requires 3 GB of memory along with 1 GB free disk space. +If not yet completed, follow the :ref:`installation` steps first. + The tutorial will cover examples on how to - configure and customise the PyPSA-Eur model and - run the ``snakemake`` workflow step by step from network creation to the solved network. -If not yet completed, follow the :ref:`installation` steps but -substitute the links for the **data bundle** and the **cutouts** with the following lightweight alternatives: - - .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517921.svg - :target: https://doi.org/10.5281/zenodo.3517921 - - .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3518020.svg - :target: https://doi.org/10.5281/zenodo.3518020 - - - **Data Bundle:** ``https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz`` (197 MB) - - - - **Cutouts:** ``https://zenodo.org/record/3518020/files/pypsa-eur-tutorial-cutouts.tar.xz`` (19 MB) - The configuration of the tutorial is included in the ``config.tutorial.yaml``. To run the tutorial, use this as your configuration file ``config.yaml``. @@ -38,6 +26,11 @@ To run the tutorial, use this as your configuration file ``config.yaml``. .../pypsa-eur % cp config.tutorial.yaml config.yaml +This configuration is set to download a reduced data set via the rules :mod:`retrieve_databundle`, +:mod:`retrieve_natura_raster`, :mod:`retrieve_cutout` totalling at less than 250 MB. +The full set of data dependencies would consume 5.3 GB. +For more information on the data dependencies of PyPSA-Eur, continue reading :ref:`data`. + How to customise PyPSA-Eur? =========================== diff --git a/scripts/retrieve_cutout.py b/scripts/retrieve_cutout.py index fb33a9d5..d6ad733f 100644 --- a/scripts/retrieve_cutout.py +++ b/scripts/retrieve_cutout.py @@ -1,19 +1,58 @@ ## Copyright 2019 Fabian Hofmann (FIAS) +""" +.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517949.svg + :target: https://doi.org/10.5281/zenodo.3517949 + +Cutouts are spatiotemporal subsets of the European weather data from the `ECMWF ERA5 `_ reanalysis dataset and the `CMSAF SARAH-2 `_ solar surface radiation dataset for the year 2013 (3.9 GB). +They have been prepared by and are for use with the `atlite `_ tool. You can either generate them yourself using the ``build_cutouts`` rule or retrieve them directly from `zenodo `_ through the rule ``retrieve_cutout`` described here. + +.. note:: + To download cutouts yourself from the `ECMWF ERA5 `_ you need to `set up the CDS API `_. + +The :ref:`tutorial` uses smaller `cutouts `_ than required for the full model (19 MB) + +.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3518020.svg + :target: https://doi.org/10.5281/zenodo.3518020 + + +**Relevant Settings** + +.. code:: yaml + + tutorial: + enable: + build_cutout: + +.. seealso:: + Documentation of the configuration file ``config.yaml`` at + :ref:`toplevel_cf` + +**Outputs** + +- ``cutouts/{cutout}``: weather data from either the `ERA5 `_ reanalysis weather dataset or `SARAH-2 `_ satellite-based historic weather data. + +.. seealso:: + For details see :mod:`build_cutout` and read the `atlite documentation `_. + +""" + import logging, os, tarfile from _helpers import progress_retrieve logger = logging.getLogger(__name__) -if snakemake.config['tutorial']: - url = "https://zenodo.org/record/3518020/files/pypsa-eur-tutorial-cutouts.tar.xz" -else: - url = "https://zenodo.org/record/3517949/files/pypsa-eur-cutouts.tar.xz" +if __name__ == "__main__": -tarball_fn = "./cutouts.tar.xz" + if snakemake.config['tutorial']: + url = "https://zenodo.org/record/3518020/files/pypsa-eur-tutorial-cutouts.tar.xz" + else: + url = "https://zenodo.org/record/3517949/files/pypsa-eur-cutouts.tar.xz" -progress_retrieve(url, tarball_fn) + tarball_fn = "./cutouts.tar.xz" -tarfile.open(tarball_fn).extractall() + progress_retrieve(url, tarball_fn) -os.remove(tarball_fn) + tarfile.open(tarball_fn).extractall() + + os.remove(tarball_fn) diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index f402e237..904a9896 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -1,19 +1,49 @@ ## Copyright 2019 Fabian Hofmann (FIAS) +""" +.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517935.svg + :target: https://doi.org/10.5281/zenodo.3517935 + +The data bundle (1.4 GB) contains common GIS datasets like NUTS3 shapes, EEZ shapes, CORINE Landcover, Natura 2000 and also electricity specific summary statistics like historic per country yearly totals of hydro generation, GDP and POP on NUTS3 levels and per-country load time-series. + +This rule downloads the data bundle from `zenodo `_ and extracts it in the ``data`` sub-directory, such that all files of the bundle are stored in the ``data/bundle`` subdirectory. + +The :ref:`tutorial` uses a smaller `data bundle `_ than required for the full model (19 MB) + +.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517921.svg + :target: https://doi.org/10.5281/zenodo.3517921 + +**Relevant Settings** + +.. code:: yaml + + tutorial: + +.. seealso:: + Documentation of the configuration file ``config.yaml`` at + :ref:`toplevel_cf` + +**Outputs** + +- ``cutouts/bundle``: input data collected from various sources + +""" import logging, os, tarfile from _helpers import progress_retrieve logger = logging.getLogger(__name__) -if snakemake.config['tutorial']: - url = "https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz" -else: - url = "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz" +if __name__ == "__main__": -tarball_fn = "./bundle.tar.xz" + if snakemake.config['tutorial']: + url = "https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz" + else: + url = "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz" -progress_retrieve(url, tarball_fn) + tarball_fn = "./bundle.tar.xz" -tarfile.open(tarball_fn).extractall('./data') + progress_retrieve(url, tarball_fn) -os.remove(tarball_fn) + tarfile.open(tarball_fn).extractall('./data') + + os.remove(tarball_fn) diff --git a/scripts/retrieve_natura_raster.py b/scripts/retrieve_natura_raster.py index fd0f9332..2974e11a 100644 --- a/scripts/retrieve_natura_raster.py +++ b/scripts/retrieve_natura_raster.py @@ -1,14 +1,42 @@ ## Copyright 2019 Fabian Hofmann (FIAS) +""" +.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3518215.svg + :target: https://doi.org/10.5281/zenodo.3518215 + +This rule, as a substitute for :mod:`build_natura_raster`, downloads an already rasterized version (`natura.tiff `_) of `Natura 2000 `_ natural protection areas to reduce computation times. The file is placed into the ``resources`` sub-directory. + +**Relevant Settings** + +.. code:: yaml + + enable: + build_natura_raster: + +.. seealso:: + Documentation of the configuration file ``config.yaml`` at + :ref:`toplevel_cf` + +**Outputs** + +- ``resources/natura.tiff``: Rasterized version of `Natura 2000 `_ natural protection areas to reduce computation times. + +.. seealso:: + For details see :mod:`build_natura_raster`. + +""" + import logging, os from _helpers import progress_retrieve logger = logging.getLogger(__name__) -d = './resources' -if not os.path.exists(d): - os.makedirs(d) -progress_retrieve("https://zenodo.org/record/3518215/files/natura.tiff", - "resources/natura.tiff") - \ No newline at end of file +if __name__ == "__main__": + + d = './resources' + if not os.path.exists(d): + os.makedirs(d) + + progress_retrieve("https://zenodo.org/record/3518215/files/natura.tiff", + "resources/natura.tiff")