Merge pull request #86 from PyPSA/workflow
Integrate data retrieving into snakemake workflow
This commit is contained in:
Fabian Neumann
GitHub
commit
99cb4a4354
@ -8,14 +8,6 @@ install:
|
||||
- sudo apt-get update --fix-missing
|
||||
- sudo apt-get install -yq --no-install-recommends curl bzip2 xz-utils git ca-certificates coinor-cbc
|
||||
|
||||
# download and extract data dependencies
|
||||
- mkdir ./resources
|
||||
- curl -L "https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz" -o "./bundle.tar.xz"
|
||||
- curl -L "https://zenodo.org/record/3518020/files/pypsa-eur-tutorial-cutouts.tar.xz" -o "./cutouts.tar.xz"
|
||||
- curl -L "https://zenodo.org/record/3518215/files/natura.tiff" -o "./resources/natura.tiff"
|
||||
- tar xJf ./bundle.tar.xz -C ./data
|
||||
- tar xJf ./cutouts.tar.xz -C .
|
||||
|
||||
# install conda
|
||||
# https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/use-conda-with-travis-ci.html
|
||||
- wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
|
||||
|
||||
47
Snakefile
47
Snakefile
@ -32,6 +32,20 @@ if config['enable']['prepare_links_p_nom']:
|
||||
# group: 'nonfeedin_preparation'
|
||||
script: 'scripts/prepare_links_p_nom.py'
|
||||
|
||||
|
||||
datafiles = ['ch_cantons.csv', 'je-e-21.03.02.xls',
|
||||
'eez/World_EEZ_v8_2014.shp', 'EIA_hydro_generation_2000_2014.csv',
|
||||
'hydro_capacities.csv', 'naturalearth/ne_10m_admin_0_countries.shp',
|
||||
'NUTS_2013_60M_SH/data/NUTS_RG_60M_2013.shp', 'nama_10r_3popgdp.tsv.gz',
|
||||
'nama_10r_3gdp.tsv.gz', 'time_series_60min_singleindex_filtered.csv',
|
||||
'corine/g250_clc06_V18_5.tif']
|
||||
if not config['tutorial']:
|
||||
datafiles.append(["data/bundle/natura/Natura2000_end2015.shp", "data/bundle/GEBCO_2014_2D.nc"])
|
||||
|
||||
rule retrieve_databundle:
|
||||
output: expand('data/bundle/{file}', file=datafiles)
|
||||
script: 'scripts/retrieve_databundle.py'
|
||||
|
||||
rule build_powerplants:
|
||||
input:
|
||||
base_network="networks/base.nc",
|
||||
@ -93,18 +107,29 @@ rule build_bus_regions:
|
||||
# group: 'nonfeedin_preparation'
|
||||
script: "scripts/build_bus_regions.py"
|
||||
|
||||
rule build_cutout:
|
||||
output: directory("cutouts/{cutout}")
|
||||
resources: mem=config['atlite'].get('nprocesses', 4) * 1000
|
||||
threads: config['atlite'].get('nprocesses', 4)
|
||||
benchmark: "benchmarks/build_cutout_{cutout}"
|
||||
# group: 'feedin_preparation'
|
||||
script: "scripts/build_cutout.py"
|
||||
if config['enable']['build_cutout']:
|
||||
rule build_cutout:
|
||||
output: directory("cutouts/{cutout}")
|
||||
resources: mem=config['atlite'].get('nprocesses', 4) * 1000
|
||||
threads: config['atlite'].get('nprocesses', 4)
|
||||
benchmark: "benchmarks/build_cutout_{cutout}"
|
||||
# group: 'feedin_preparation'
|
||||
script: "scripts/build_cutout.py"
|
||||
else:
|
||||
rule retrieve_cutout:
|
||||
output: directory("cutouts/{cutout}")
|
||||
script: 'scripts/retrieve_cutout.py'
|
||||
|
||||
rule build_natura_raster:
|
||||
input: "data/bundle/natura/Natura2000_end2015.shp"
|
||||
output: "resources/natura.tiff"
|
||||
script: "scripts/build_natura_raster.py"
|
||||
|
||||
if config['enable']['build_natura_raster']:
|
||||
rule build_natura_raster:
|
||||
input: "data/bundle/natura/Natura2000_end2015.shp"
|
||||
output: "resources/natura.tiff"
|
||||
script: "scripts/build_natura_raster.py"
|
||||
else:
|
||||
rule retrieve_natura_raster:
|
||||
output: "resources/natura.tiff"
|
||||
script: 'scripts/retrieve_natura_raster.py'
|
||||
|
||||
rule build_renewable_profiles:
|
||||
input:
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
version: 0.1
|
||||
tutorial: false
|
||||
logging_level: INFO
|
||||
|
||||
summary_dir: results
|
||||
@ -19,6 +20,8 @@ snapshots:
|
||||
|
||||
enable:
|
||||
prepare_links_p_nom: false
|
||||
build_cutout: false
|
||||
build_natura_raster: false
|
||||
|
||||
electricity:
|
||||
voltages: [220., 300., 380.]
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
version: 0.1
|
||||
tutorial: true
|
||||
logging_level: INFO
|
||||
|
||||
summary_dir: results
|
||||
@ -19,6 +20,8 @@ snapshots:
|
||||
|
||||
enable:
|
||||
prepare_links_p_nom: false
|
||||
build_cutout: false
|
||||
build_natura_raster: false
|
||||
|
||||
electricity:
|
||||
voltages: [220., 300., 380.]
|
||||
|
||||
@ -1,8 +1,10 @@
|
||||
,Unit,Values,Description
|
||||
version,--,0.1,"Version of PyPSA-Eur"
|
||||
tutorial,bool,"{true, false}","Switch to retrieve the tutorial data set instead of the full data set."
|
||||
logging_level,--,"Any of {'INFO', 'WARNING', 'ERROR'}","Restrict console outputs to all infos, warning or errors only"
|
||||
summary_dir,--,"e.g. 'results'","Directory into which results are written."
|
||||
countries,--,"Subset of {'AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'}","European countries defined by their `Two-letter country codes (ISO 3166-1) <https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2>`_ which should be included in the energy system model."
|
||||
enable,,,
|
||||
-- powerplantmatching,bool,"{true, false}","Switch to retrieve currently existing conventional power plant capacities matched from multiple sources by using `powerplantmatching <https://www.github.com/FRESNA/powerplantmatching>`_."
|
||||
-- prepare_links_p_nom,bool,"{true, false}","Switch to retrieve current HVDC projects from `Wikipedia <https://en.wikipedia.org/wiki/List_of_HVDC_projects>`_"
|
||||
-- prepare_links_p_nom,bool,"{true, false}","Switch to retrieve current HVDC projects from `Wikipedia <https://en.wikipedia.org/wiki/List_of_HVDC_projects>`_"
|
||||
-- build_cutout,bool,"{true, false}","Switch to build cutouts via the rule :mod:`build_cutout` or whether to retrieve the data from zenodo with :mod:`retrieve_cutout`."
|
||||
-- build_natura_raster,bool,"{true, false}","Switch to create raster ``natura.tiff`` via the rule :mod:`build_natura_raster` or whether to retrieve ``natura.tiff`` from zenodo with :mod:`retrieve_natura_raster`."
|
||||
|
@ -43,70 +43,7 @@ The environment can be installed and activated using
|
||||
|
||||
.. note::
|
||||
Note that activation is local to the currently open shell!
|
||||
After opening a new terminal window, one needs to reissue the second command!
|
||||
|
||||
.. _data:
|
||||
|
||||
Download Data Dependencies
|
||||
==============================
|
||||
|
||||
Not all data dependencies are shipped with the git repository,
|
||||
since git is not suited for handling large changing files.
|
||||
Instead we provide separate data bundles which can be obtained
|
||||
using the described shell commands or by downloading and
|
||||
extracting them manually in the locations outlined below.
|
||||
|
||||
.. note::
|
||||
|
||||
The :ref:`tutorial` uses smaller data bundles than required for the full model.
|
||||
To start with the tutorial, substitute with the links below using the following alternatives:
|
||||
|
||||
.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517921.svg
|
||||
:target: https://doi.org/10.5281/zenodo.3517921
|
||||
|
||||
.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3518020.svg
|
||||
:target: https://doi.org/10.5281/zenodo.3518020
|
||||
|
||||
- **Data Bundle:** ``https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz`` (197 MB)
|
||||
- **Cutouts:** ``https://zenodo.org/record/3518020/files/pypsa-eur-tutorial-cutouts.tar.xz`` (19 MB)
|
||||
|
||||
.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517935.svg
|
||||
:target: https://doi.org/10.5281/zenodo.3517935
|
||||
|
||||
1. **Data Bundle:** `pypsa-eur-data-bundle.tar.xz <https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz>`_ (1.4 GB) contains common GIS datasets like NUTS3 shapes, EEZ shapes, CORINE Landcover, Natura 2000 and also electricity specific summary statistics like historic per country yearly totals of hydro generation, GDP and POP on NUTS3 levels and per-country load time-series. It should be extracted in the ``data`` sub-directory, such that all files of the bundle are stored in the ``data/bundle`` subdirectory)
|
||||
|
||||
.. code:: bash
|
||||
|
||||
.../pypsa-eur/data % curl -OL "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz"
|
||||
|
||||
.../pypsa-eur/data % tar xJf pypsa-eur-data-bundle.tar.xz
|
||||
|
||||
.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517949.svg
|
||||
:target: https://doi.org/10.5281/zenodo.3517949
|
||||
|
||||
2. **Cutouts:** `pypsa-eur-cutouts.tar.xz <https://zenodo.org/record/3517949/files/pypsa-eur-cutouts.tar.xz>`_ (3.9 GB) are spatiotemporal subsets of the European weather data from the `ECMWF ERA5 <https://software.ecmwf.int/wiki/display/CKB/ERA5+data+documentation>`_ reanalysis dataset and the `CMSAF SARAH-2 <https://wui.cmsaf.eu/safira/action/viewDoiDetails?acronym=SARAH_V002>`_ solar surface radiation dataset for the year 2013. They have been prepared by and are for use with the `atlite <https://github.com/PyPSA/atlite>`_ tool. You can either generate them yourself using the ``build_cutouts`` rule or extract them directly into the ``pypsa-eur`` directory. To download cutouts yourself you need to `set up the CDS API <https://cds.climate.copernicus.eu/api-how-to>`_. For more details read the `atlite documentation <https://atlite.readthedocs.io>`_. For beginners, extracting the bundle is recommended:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
.../pypsa-eur % curl -OL "https://zenodo.org/record/3517949/files/pypsa-eur-cutouts.tar.xz"
|
||||
|
||||
.../pypsa-eur % tar xJf pypsa-eur-cutouts.tar.xz
|
||||
|
||||
.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3518215.svg
|
||||
:target: https://doi.org/10.5281/zenodo.3518215
|
||||
|
||||
3. **Natura:** Optionally, you can download a rasterized version of the NATURA dataset `natura.tiff <https://zenodo.org/record/3518215/files/natura.tiff>`_ and put it into the ``resources`` sub-directory. If you don't, it will be generated automatically, which is a time-consuming process.
|
||||
|
||||
.. code:: bash
|
||||
|
||||
.../pypsa-eur % curl -L "https://zenodo.org/record/3518215/files/natura.tiff" -o "resources/natura.tiff"
|
||||
|
||||
|
||||
4. **Remove Archives:** Optionally, if you want to save disk space, you can delete ``data/pypsa-eur-data-bundle.tar.xz`` and ``pypsa-eur-cutouts.tar.xz`` once extracting the bundles is complete. E.g.
|
||||
|
||||
.. code:: bash
|
||||
|
||||
.../pypsa-eur % rm -rf data/pypsa-eur-data-bundle.tar.xz pypsa-eur-cutouts.tar.xz
|
||||
After opening a new terminal window, one needs to reissue the second command!
|
||||
|
||||
Install a Solver
|
||||
================
|
||||
|
||||
@ -3,7 +3,12 @@ Preparing Networks
|
||||
##########################################
|
||||
|
||||
The preparation process of the PyPSA-Eur energy system model consists of a group of ``snakemake``
|
||||
rules which are briefly outlined and explained in detail in the sections below:
|
||||
rules which are briefly outlined and explained in detail in the sections below.
|
||||
|
||||
Not all data dependencies are shipped with the git repository.
|
||||
Instead we provide separate data bundles which can be obtained
|
||||
using the ``retrieve*`` rules (:ref:`data`).
|
||||
Having downloaded the necessary data,
|
||||
|
||||
- :mod:`build_shapes` generates GeoJSON files with shapes of the countries, exclusive economic zones and `NUTS3 <https://en.wikipedia.org/wiki/Nomenclature_of_Territorial_Units_for_Statistics>`_ areas.
|
||||
- :mod:`build_cutout` prepares smaller weather data portions from `ERA5 <https://www.ecmwf.int/en/forecasts/datasets/reanalysis-datasets/era5>`_ for cutout ``europe-2013-era5`` and SARAH for cutout ``europe-2013-sarah``.
|
||||
@ -17,6 +22,7 @@ With these and the externally extracted ENTSO-E online map topology
|
||||
Then the process continues by calculating conventional power plant capacities, potentials, and per-unit availability time series for variable renewable energy carriers and hydro power plants with the following rules:
|
||||
|
||||
- :mod:`build_powerplants` for today's thermal power plant capacities using `powerplantmatching <https://github.com/FRESNA/powerplantmatching>`_ allocating these to the closest substation for each powerplant,
|
||||
- :mod:`build_natura_raster` for rasterising NATURA2000 natural protection areas,
|
||||
- :mod:`build_renewable_profiles` for the hourly capacity factors and installation potentials constrained by land-use in each substation's Voronoi cell for PV, onshore and offshore wind, and
|
||||
- :mod:`build_hydro_profile` for the hourly per-unit hydro power availability time series.
|
||||
|
||||
@ -26,8 +32,10 @@ together into a detailed PyPSA network stored in ``networks/elec.nc``.
|
||||
.. toctree::
|
||||
:caption: Overview
|
||||
|
||||
preparation/retrieve
|
||||
preparation/build_shapes
|
||||
preparation/build_cutout
|
||||
preparation/build_natura_raster
|
||||
preparation/prepare_links_p_nom
|
||||
preparation/base_network
|
||||
preparation/build_bus_regions
|
||||
|
||||
24
doc/preparation/retrieve.rst
Normal file
24
doc/preparation/retrieve.rst
Normal file
@ -0,0 +1,24 @@
|
||||
.. _data:
|
||||
|
||||
Rules ``retrieve*``
|
||||
=============================
|
||||
|
||||
Not all data dependencies are shipped with the git repository,
|
||||
since git is not suited for handling large changing files.
|
||||
Instead we provide separate data bundles which can be obtained
|
||||
using the ``retrieve*`` rules.
|
||||
|
||||
Rule ``retrieve_databundle``
|
||||
----------------------------
|
||||
|
||||
.. automodule:: retrieve_databundle
|
||||
|
||||
Rule ``retrieve_cutout``
|
||||
------------------------
|
||||
|
||||
.. automodule:: retrieve_cutout
|
||||
|
||||
Rule ``retrieve_natura_raster``
|
||||
-------------------------------
|
||||
|
||||
.. automodule:: retrieve_natura_raster
|
||||
@ -12,25 +12,13 @@ which allows the user to explore most of its functionalities on a local machine.
|
||||
It takes approximately five minutes to complete and
|
||||
requires 3 GB of memory along with 1 GB free disk space.
|
||||
|
||||
If not yet completed, follow the :ref:`installation` steps first.
|
||||
|
||||
The tutorial will cover examples on how to
|
||||
|
||||
- configure and customise the PyPSA-Eur model and
|
||||
- run the ``snakemake`` workflow step by step from network creation to the solved network.
|
||||
|
||||
If not yet completed, follow the :ref:`installation` steps but
|
||||
substitute the links for the **data bundle** and the **cutouts** with the following lightweight alternatives:
|
||||
|
||||
.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517921.svg
|
||||
:target: https://doi.org/10.5281/zenodo.3517921
|
||||
|
||||
.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3518020.svg
|
||||
:target: https://doi.org/10.5281/zenodo.3518020
|
||||
|
||||
- **Data Bundle:** ``https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz`` (197 MB)
|
||||
|
||||
|
||||
- **Cutouts:** ``https://zenodo.org/record/3518020/files/pypsa-eur-tutorial-cutouts.tar.xz`` (19 MB)
|
||||
|
||||
The configuration of the tutorial is included in the ``config.tutorial.yaml``.
|
||||
To run the tutorial, use this as your configuration file ``config.yaml``.
|
||||
|
||||
@ -38,6 +26,11 @@ To run the tutorial, use this as your configuration file ``config.yaml``.
|
||||
|
||||
.../pypsa-eur % cp config.tutorial.yaml config.yaml
|
||||
|
||||
This configuration is set to download a reduced data set via the rules :mod:`retrieve_databundle`,
|
||||
:mod:`retrieve_natura_raster`, :mod:`retrieve_cutout` totalling at less than 250 MB.
|
||||
The full set of data dependencies would consume 5.3 GB.
|
||||
For more information on the data dependencies of PyPSA-Eur, continue reading :ref:`data`.
|
||||
|
||||
How to customise PyPSA-Eur?
|
||||
===========================
|
||||
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from six import iteritems, iterkeys, itervalues
|
||||
from six import iterkeys, itervalues
|
||||
import urllib
|
||||
from progressbar import ProgressBar
|
||||
|
||||
import pypsa
|
||||
|
||||
@ -104,3 +105,12 @@ def aggregate_costs(n, flatten=False, opts=None, existing_only=False):
|
||||
)
|
||||
|
||||
return costs
|
||||
|
||||
def progress_retrieve(url, file):
|
||||
pbar = ProgressBar(0, 100)
|
||||
|
||||
def dlProgress(count, blockSize, totalSize):
|
||||
pbar.update( int(count * blockSize * 100 / totalSize) )
|
||||
|
||||
urllib.request.urlretrieve(url, file, reporthook=dlProgress)
|
||||
|
||||
|
||||
@ -18,26 +18,26 @@ Relevant Settings
|
||||
types:
|
||||
s_max_pu:
|
||||
under_construction:
|
||||
|
||||
|
||||
links:
|
||||
p_max_pu:
|
||||
under_construction:
|
||||
include_tyndp:
|
||||
|
||||
|
||||
transformers:
|
||||
x:
|
||||
s_nom:
|
||||
type:
|
||||
|
||||
.. seealso::
|
||||
.. seealso::
|
||||
Documentation of the configuration file ``config.yaml`` at
|
||||
:ref:`snapshots_cf`, :ref:`toplevel_cf`, :ref:`electricity_cf`, :ref:`load_cf`,
|
||||
:ref:`lines_cf`, :ref:`links_cf`, :ref:`transformers_cf`
|
||||
|
||||
|
||||
Inputs
|
||||
------
|
||||
|
||||
- ``data/entsoegridkit``: Extract from the geographical vector data of the online `ENTSO-E Interactive Map <https://www.entsoe.eu/data/map/>`_ by the `GridKit <https://github.com/pypsa/gridkit>`_ toolkit.
|
||||
- ``data/entsoegridkit``: Extract from the geographical vector data of the online `ENTSO-E Interactive Map <https://www.entsoe.eu/data/map/>`_ by the `GridKit <https://github.com/pypsa/gridkit>`_ toolkit.
|
||||
- ``data/parameter_corrections.yaml``: Corrections for ``data/entsoegridkit``
|
||||
- ``data/links_p_nom.csv``: confer :ref:`links`
|
||||
- ``data/links_tyndp.csv``: List of projects in the `TYNDP 2018 <https://tyndp.entsoe.eu/tyndp2018/>`_ that are at least *in permitting* with fields for start- and endpoint (names and coordinates), length, capacity, construction status, and project reference ID.
|
||||
@ -236,7 +236,7 @@ def _add_links_from_tyndp(buses, links):
|
||||
|
||||
links_tyndp.index = "T" + links_tyndp.index.astype(str)
|
||||
|
||||
return buses, links.append(links_tyndp)
|
||||
return buses, links.append(links_tyndp, sort=True)
|
||||
|
||||
def _load_lines_from_eg(buses):
|
||||
lines = (pd.read_csv(snakemake.input.eg_lines, quotechar="'", true_values='t', false_values='f',
|
||||
@ -469,7 +469,7 @@ def _replace_b2b_converter_at_country_border_by_link(n):
|
||||
|
||||
def _set_links_underwater_fraction(n):
|
||||
if n.links.empty: return
|
||||
|
||||
|
||||
if not hasattr(n.links, 'geometry'):
|
||||
n.links['underwater_fraction'] = 0.
|
||||
else:
|
||||
|
||||
@ -8,7 +8,7 @@ Relevant Settings
|
||||
|
||||
countries:
|
||||
|
||||
.. seealso::
|
||||
.. seealso::
|
||||
Documentation of the configuration file ``config.yaml`` at
|
||||
:ref:`toplevel_cf`
|
||||
|
||||
@ -52,9 +52,9 @@ Outputs
|
||||
|
||||
.. image:: ../img/europe_shape.png
|
||||
:scale: 33 %
|
||||
|
||||
|
||||
- ``resources/nuts3_shapes.geojson``: NUTS3 shapes out of country selection including population and GDP data.
|
||||
|
||||
|
||||
.. image:: ../img/nuts3_shapes.png
|
||||
:scale: 33 %
|
||||
|
||||
@ -182,7 +182,7 @@ def nuts3(country_shapes):
|
||||
manual['geometry'] = manual['country'].map(country_shapes)
|
||||
manual = manual.dropna()
|
||||
|
||||
df = df.append(manual)
|
||||
df = df.append(manual, sort=False)
|
||||
|
||||
df.loc['ME000', 'pop'] = 650.
|
||||
|
||||
|
||||
58
scripts/retrieve_cutout.py
Normal file
58
scripts/retrieve_cutout.py
Normal file
@ -0,0 +1,58 @@
|
||||
## Copyright 2019 Fabian Hofmann (FIAS)
|
||||
|
||||
"""
|
||||
.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517949.svg
|
||||
:target: https://doi.org/10.5281/zenodo.3517949
|
||||
|
||||
Cutouts are spatiotemporal subsets of the European weather data from the `ECMWF ERA5 <https://software.ecmwf.int/wiki/display/CKB/ERA5+data+documentation>`_ reanalysis dataset and the `CMSAF SARAH-2 <https://wui.cmsaf.eu/safira/action/viewDoiDetails?acronym=SARAH_V002>`_ solar surface radiation dataset for the year 2013 (3.9 GB).
|
||||
They have been prepared by and are for use with the `atlite <https://github.com/PyPSA/atlite>`_ tool. You can either generate them yourself using the ``build_cutouts`` rule or retrieve them directly from `zenodo <https://doi.org/10.5281/zenodo.3517949>`_ through the rule ``retrieve_cutout`` described here.
|
||||
|
||||
.. note::
|
||||
To download cutouts yourself from the `ECMWF ERA5 <https://software.ecmwf.int/wiki/display/CKB/ERA5+data+documentation>`_ you need to `set up the CDS API <https://cds.climate.copernicus.eu/api-how-to>`_.
|
||||
|
||||
The :ref:`tutorial` uses smaller `cutouts <https://zenodo.org/record/3518020/files/pypsa-eur-tutorial-cutouts.tar.xz>`_ than required for the full model (19 MB)
|
||||
|
||||
.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3518020.svg
|
||||
:target: https://doi.org/10.5281/zenodo.3518020
|
||||
|
||||
|
||||
**Relevant Settings**
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
tutorial:
|
||||
enable:
|
||||
build_cutout:
|
||||
|
||||
.. seealso::
|
||||
Documentation of the configuration file ``config.yaml`` at
|
||||
:ref:`toplevel_cf`
|
||||
|
||||
**Outputs**
|
||||
|
||||
- ``cutouts/{cutout}``: weather data from either the `ERA5 <https://www.ecmwf.int/en/forecasts/datasets/reanalysis-datasets/era5>`_ reanalysis weather dataset or `SARAH-2 <https://wui.cmsaf.eu/safira/action/viewProduktSearch>`_ satellite-based historic weather data.
|
||||
|
||||
.. seealso::
|
||||
For details see :mod:`build_cutout` and read the `atlite documentation <https://atlite.readthedocs.io>`_.
|
||||
|
||||
"""
|
||||
|
||||
import logging, os, tarfile
|
||||
from _helpers import progress_retrieve
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
if snakemake.config['tutorial']:
|
||||
url = "https://zenodo.org/record/3518020/files/pypsa-eur-tutorial-cutouts.tar.xz"
|
||||
else:
|
||||
url = "https://zenodo.org/record/3517949/files/pypsa-eur-cutouts.tar.xz"
|
||||
|
||||
tarball_fn = "./cutouts.tar.xz"
|
||||
|
||||
progress_retrieve(url, tarball_fn)
|
||||
|
||||
tarfile.open(tarball_fn).extractall()
|
||||
|
||||
os.remove(tarball_fn)
|
||||
49
scripts/retrieve_databundle.py
Normal file
49
scripts/retrieve_databundle.py
Normal file
@ -0,0 +1,49 @@
|
||||
## Copyright 2019 Fabian Hofmann (FIAS)
|
||||
"""
|
||||
.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517935.svg
|
||||
:target: https://doi.org/10.5281/zenodo.3517935
|
||||
|
||||
The data bundle (1.4 GB) contains common GIS datasets like NUTS3 shapes, EEZ shapes, CORINE Landcover, Natura 2000 and also electricity specific summary statistics like historic per country yearly totals of hydro generation, GDP and POP on NUTS3 levels and per-country load time-series.
|
||||
|
||||
This rule downloads the data bundle from `zenodo <https://doi.org/10.5281/zenodo.3517935>`_ and extracts it in the ``data`` sub-directory, such that all files of the bundle are stored in the ``data/bundle`` subdirectory.
|
||||
|
||||
The :ref:`tutorial` uses a smaller `data bundle <https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz>`_ than required for the full model (19 MB)
|
||||
|
||||
.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517921.svg
|
||||
:target: https://doi.org/10.5281/zenodo.3517921
|
||||
|
||||
**Relevant Settings**
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
tutorial:
|
||||
|
||||
.. seealso::
|
||||
Documentation of the configuration file ``config.yaml`` at
|
||||
:ref:`toplevel_cf`
|
||||
|
||||
**Outputs**
|
||||
|
||||
- ``cutouts/bundle``: input data collected from various sources
|
||||
|
||||
"""
|
||||
|
||||
import logging, os, tarfile
|
||||
from _helpers import progress_retrieve
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
if snakemake.config['tutorial']:
|
||||
url = "https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz"
|
||||
else:
|
||||
url = "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz"
|
||||
|
||||
tarball_fn = "./bundle.tar.xz"
|
||||
|
||||
progress_retrieve(url, tarball_fn)
|
||||
|
||||
tarfile.open(tarball_fn).extractall('./data')
|
||||
|
||||
os.remove(tarball_fn)
|
||||
42
scripts/retrieve_natura_raster.py
Normal file
42
scripts/retrieve_natura_raster.py
Normal file
@ -0,0 +1,42 @@
|
||||
## Copyright 2019 Fabian Hofmann (FIAS)
|
||||
|
||||
"""
|
||||
.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3518215.svg
|
||||
:target: https://doi.org/10.5281/zenodo.3518215
|
||||
|
||||
This rule, as a substitute for :mod:`build_natura_raster`, downloads an already rasterized version (`natura.tiff <https://zenodo.org/record/3518215/files/natura.tiff>`_) of `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas to reduce computation times. The file is placed into the ``resources`` sub-directory.
|
||||
|
||||
**Relevant Settings**
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
enable:
|
||||
build_natura_raster:
|
||||
|
||||
.. seealso::
|
||||
Documentation of the configuration file ``config.yaml`` at
|
||||
:ref:`toplevel_cf`
|
||||
|
||||
**Outputs**
|
||||
|
||||
- ``resources/natura.tiff``: Rasterized version of `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas to reduce computation times.
|
||||
|
||||
.. seealso::
|
||||
For details see :mod:`build_natura_raster`.
|
||||
|
||||
"""
|
||||
|
||||
import logging, os
|
||||
from _helpers import progress_retrieve
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
d = './resources'
|
||||
if not os.path.exists(d):
|
||||
os.makedirs(d)
|
||||
|
||||
progress_retrieve("https://zenodo.org/record/3518215/files/natura.tiff",
|
||||
"resources/natura.tiff")
|
||||
@ -1,4 +1,5 @@
|
||||
version: 0.1
|
||||
tutorial: true
|
||||
logging_level: INFO
|
||||
|
||||
summary_dir: results
|
||||
@ -19,7 +20,9 @@ snapshots:
|
||||
|
||||
enable:
|
||||
prepare_links_p_nom: false
|
||||
|
||||
build_cutout: false
|
||||
build_natura_raster: false
|
||||
|
||||
electricity:
|
||||
voltages: [220., 300., 380.]
|
||||
co2limit: 100.e+6
|
||||
|
||||
Loading…
Reference in New Issue
Block a user