Merge branch 'master' into eu-energy-security
This commit is contained in:
Fabian Neumann
GitHub
commit
5ca231824e
21
.github/workflows/ci.yaml
vendored
21
.github/workflows/ci.yaml
vendored
@ -13,13 +13,12 @@ on:
|
||||
branches:
|
||||
- master
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
schedule:
|
||||
- cron: "0 5 * * TUE"
|
||||
|
||||
env:
|
||||
CACHE_NUMBER: 1 # Change this value to manually reset the environment cache
|
||||
CONDA_CACHE_NUMBER: 1 # Change this value to manually reset the environment cache
|
||||
DATA_CACHE_NUMBER: 1
|
||||
|
||||
jobs:
|
||||
build:
|
||||
@ -67,15 +66,25 @@ jobs:
|
||||
activate-environment: pypsa-eur
|
||||
use-mamba: true
|
||||
|
||||
- name: Set cache date
|
||||
run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV
|
||||
- name: Set cache dates
|
||||
run: |
|
||||
echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV
|
||||
echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV
|
||||
|
||||
- name: Cache data and cutouts folders
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: |
|
||||
data
|
||||
cutouts
|
||||
key: data-cutouts-${{ env.WEEK }}-${{ env.DATA_CACHE_NUMBER }}
|
||||
|
||||
- name: Create environment cache
|
||||
uses: actions/cache@v2
|
||||
id: cache
|
||||
with:
|
||||
path: ${{ matrix.prefix }}
|
||||
key: ${{ matrix.label }}-conda-${{ hashFiles('envs/environment.yaml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
|
||||
key: ${{ matrix.label }}-conda-${{ hashFiles('envs/environment.yaml') }}-${{ env.DATE }}-${{ env.CONDA_CACHE_NUMBER }}
|
||||
|
||||
- name: Update environment due to outdated or unavailable cache
|
||||
run: mamba env update -n pypsa-eur -f envs/environment.yaml
|
||||
|
||||
27
Snakefile
27
Snakefile
@ -66,6 +66,12 @@ if config['enable'].get('retrieve_databundle', True):
|
||||
script: 'scripts/retrieve_databundle.py'
|
||||
|
||||
|
||||
rule retrieve_natura_data:
|
||||
input: HTTP.remote("sdi.eea.europa.eu/datashare/s/H6QGCybMdLLnywo/download", additional_request_string="?path=%2FNatura2000_end2020_gpkg&files=Natura2000_end2020.gpkg", static=True)
|
||||
output: "data/Natura2000_end2020.gpkg"
|
||||
run: move(input[0], output[0])
|
||||
|
||||
|
||||
rule retrieve_load_data:
|
||||
input: HTTP.remote("data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv", keep_local=True, static=True)
|
||||
output: "data/load_raw.csv"
|
||||
@ -164,28 +170,13 @@ if config['enable'].get('retrieve_cutout', True):
|
||||
run: move(input[0], output[0])
|
||||
|
||||
|
||||
if config['enable'].get('build_natura_raster', False):
|
||||
rule build_natura_raster:
|
||||
input:
|
||||
natura="data/bundle/natura/Natura2000_end2015.shp",
|
||||
cutouts=expand("cutouts/{cutouts}.nc", **config['atlite'])
|
||||
output: "resources/natura.tiff"
|
||||
log: "logs/build_natura_raster.log"
|
||||
script: "scripts/build_natura_raster.py"
|
||||
|
||||
|
||||
if config['enable'].get('retrieve_natura_raster', True):
|
||||
rule retrieve_natura_raster:
|
||||
input: HTTP.remote("zenodo.org/record/4706686/files/natura.tiff", keep_local=True, static=True)
|
||||
output: "resources/natura.tiff"
|
||||
run: move(input[0], output[0])
|
||||
|
||||
|
||||
rule build_renewable_profiles:
|
||||
input:
|
||||
base_network="networks/base.nc",
|
||||
corine="data/bundle/corine/g250_clc06_V18_5.tif",
|
||||
natura="resources/natura.tiff",
|
||||
natura=lambda w: ("data/Natura2000_end2020.gpkg"
|
||||
if config["renewable"][w.technology]["natura"]
|
||||
else []),
|
||||
gebco=lambda w: ("data/bundle/GEBCO_2014_2D.nc"
|
||||
if "max_depth" in config["renewable"][w.technology].keys()
|
||||
else []),
|
||||
|
||||
@ -31,8 +31,6 @@ enable:
|
||||
retrieve_databundle: true
|
||||
build_cutout: false
|
||||
retrieve_cutout: true
|
||||
build_natura_raster: false
|
||||
retrieve_natura_raster: true
|
||||
custom_busmap: false
|
||||
|
||||
electricity:
|
||||
|
||||
@ -32,8 +32,6 @@ enable:
|
||||
retrieve_databundle: true
|
||||
build_cutout: false
|
||||
retrieve_cutout: true
|
||||
build_natura_raster: false
|
||||
retrieve_natura_raster: true
|
||||
custom_busmap: false
|
||||
|
||||
electricity:
|
||||
@ -78,7 +76,7 @@ renewable:
|
||||
24, 25, 26, 27, 28, 29, 31, 32]
|
||||
distance: 1000
|
||||
distance_grid_codes: [1, 2, 3, 4, 5, 6]
|
||||
natura: true
|
||||
natura: false
|
||||
potential: simple # or conservative
|
||||
clip_p_max_pu: 1.e-2
|
||||
offwind-ac:
|
||||
@ -89,7 +87,7 @@ renewable:
|
||||
capacity_per_sqkm: 3
|
||||
# correction_factor: 0.93
|
||||
corine: [44, 255]
|
||||
natura: true
|
||||
natura: false
|
||||
max_shore_distance: 30000
|
||||
potential: simple # or conservative
|
||||
clip_p_max_pu: 1.e-2
|
||||
@ -102,7 +100,7 @@ renewable:
|
||||
capacity_per_sqkm: 3
|
||||
# correction_factor: 0.93
|
||||
corine: [44, 255]
|
||||
natura: true
|
||||
natura: false
|
||||
min_shore_distance: 30000
|
||||
potential: simple # or conservative
|
||||
clip_p_max_pu: 1.e-2
|
||||
@ -124,7 +122,7 @@ renewable:
|
||||
# correction_factor: 0.854337
|
||||
corine: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
|
||||
14, 15, 16, 17, 18, 19, 20, 26, 31, 32]
|
||||
natura: true
|
||||
natura: false
|
||||
potential: simple # or conservative
|
||||
clip_p_max_pu: 1.e-2
|
||||
|
||||
|
||||
@ -12,6 +12,4 @@ enable,,,
|
||||
-- retrieve_databundle,bool,"{true, false}","Switch to retrieve databundle from zenodo via the rule :mod:`retrieve_databundle` or whether to keep a custom databundle located in the corresponding folder."
|
||||
-- build_cutout,bool,"{true, false}","Switch to enable the building of cutouts via the rule :mod:`build_cutout`."
|
||||
-- retrieve_cutout,bool,"{true, false}","Switch to enable the retrieval of cutouts from zenodo with :mod:`retrieve_cutout`."
|
||||
-- build_natura_raster,bool,"{true, false}","Switch to enable the creation of the raster ``natura.tiff`` via the rule :mod:`build_natura_raster`."
|
||||
-- retrieve_natura_raster,bool,"{true, false}","Switch to enable the retrieval of ``natura.tiff`` from zenodo with :mod:`retrieve_natura_raster`."
|
||||
-- custom_busmap,bool,"{true, false}","Switch to enable the use of custom busmaps in rule :mod:`cluster_network`. If activated the rule looks for provided busmaps at ``data/custom_busmap_elec_s{simpl}_{clusters}.csv`` which should have the same format as ``resources/busmap_elec_s{simpl}_{clusters}.csv``, i.e. the index should contain the buses of ``networks/elec_s{simpl}.nc``."
|
||||
|
||||
|
@ -27,7 +27,6 @@ With these and the externally extracted ENTSO-E online map topology
|
||||
Then the process continues by calculating conventional power plant capacities, potentials, and per-unit availability time series for variable renewable energy carriers and hydro power plants with the following rules:
|
||||
|
||||
- :mod:`build_powerplants` for today's thermal power plant capacities using `powerplantmatching <https://github.com/FRESNA/powerplantmatching>`_ allocating these to the closest substation for each powerplant,
|
||||
- :mod:`build_natura_raster` for rasterising NATURA2000 natural protection areas,
|
||||
- :mod:`build_renewable_profiles` for the hourly capacity factors and installation potentials constrained by land-use in each substation's Voronoi cell for PV, onshore and offshore wind, and
|
||||
- :mod:`build_hydro_profile` for the hourly per-unit hydro power availability time series.
|
||||
|
||||
@ -41,7 +40,6 @@ together into a detailed PyPSA network stored in ``networks/elec.nc``.
|
||||
preparation/build_shapes
|
||||
preparation/build_load_data
|
||||
preparation/build_cutout
|
||||
preparation/build_natura_raster
|
||||
preparation/prepare_links_p_nom
|
||||
preparation/base_network
|
||||
preparation/build_bus_regions
|
||||
|
||||
@ -1,39 +0,0 @@
|
||||
..
|
||||
SPDX-FileCopyrightText: 2019-2020 The PyPSA-Eur Authors
|
||||
|
||||
SPDX-License-Identifier: CC-BY-4.0
|
||||
|
||||
.. _natura:
|
||||
|
||||
Rule ``build_natura_raster``
|
||||
===============================
|
||||
|
||||
.. graphviz::
|
||||
:align: center
|
||||
|
||||
digraph snakemake_dag {
|
||||
graph [bgcolor=white,
|
||||
margin=0,
|
||||
size="8,5"
|
||||
];
|
||||
node [fontname=sans,
|
||||
fontsize=10,
|
||||
penwidth=2,
|
||||
shape=box,
|
||||
style=rounded
|
||||
];
|
||||
edge [color=grey,
|
||||
penwidth=2
|
||||
];
|
||||
9 [color="0.22 0.6 0.85",
|
||||
label=build_renewable_profiles];
|
||||
12 [color="0.31 0.6 0.85",
|
||||
fillcolor=gray,
|
||||
label=build_natura_raster,
|
||||
style=filled];
|
||||
12 -> 9;
|
||||
}
|
||||
|
||||
|
|
||||
|
||||
.. automodule:: build_natura_raster
|
||||
@ -41,9 +41,6 @@ Rule ``build_renewable_profiles``
|
||||
8 [color="0.00 0.6 0.85",
|
||||
label=build_shapes];
|
||||
8 -> 9;
|
||||
12 [color="0.31 0.6 0.85",
|
||||
label=build_natura_raster];
|
||||
12 -> 9;
|
||||
13 [color="0.56 0.6 0.85",
|
||||
label=build_cutout];
|
||||
13 -> 9;
|
||||
|
||||
@ -50,30 +50,3 @@ The :ref:`tutorial` uses a smaller cutout than required for the full model (30 M
|
||||
|
||||
.. seealso::
|
||||
For details see :mod:`build_cutout` and read the `atlite documentation <https://atlite.readthedocs.io>`_.
|
||||
|
||||
|
||||
Rule ``retrieve_natura_raster``
|
||||
-------------------------------
|
||||
|
||||
.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.4706686.svg
|
||||
:target: https://doi.org/10.5281/zenodo.4706686
|
||||
|
||||
This rule, as a substitute for :mod:`build_natura_raster`, downloads an already rasterized version (`natura.tiff <https://zenodo.org/record/4706686/files/natura.tiff>`_) of `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas to reduce computation times. The file is placed into the ``resources`` sub-directory.
|
||||
|
||||
**Relevant Settings**
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
enable:
|
||||
build_natura_raster:
|
||||
|
||||
.. seealso::
|
||||
Documentation of the configuration file ``config.yaml`` at
|
||||
:ref:`toplevel_cf`
|
||||
|
||||
**Outputs**
|
||||
|
||||
- ``resources/natura.tiff``: Rasterized version of `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas to reduce computation times.
|
||||
|
||||
.. seealso::
|
||||
For details see :mod:`build_natura_raster`.
|
||||
|
||||
@ -79,6 +79,12 @@ Upcoming Release
|
||||
|
||||
* Update rasterio version to correctly calculate exclusion raster
|
||||
|
||||
* Remove rules to build or retrieve rasterized NATURA 2000 dataset. Renewable potential calculation now directly uses the shapefiles.
|
||||
|
||||
* Cache data and cutouts folders. This cache will be updated weekly.
|
||||
|
||||
* Add rule to automatically retrieve Natura2000 natural protection areas. Switch of file format to GPKG.
|
||||
|
||||
|
||||
Synchronisation Release - Ukraine and Moldova (17th March 2022)
|
||||
===============================================================
|
||||
@ -115,7 +121,6 @@ This release is not on the ``master`` branch. It can be used with
|
||||
git checkout synchronisation-release
|
||||
|
||||
|
||||
|
||||
PyPSA-Eur 0.4.0 (22th September 2021)
|
||||
=====================================
|
||||
|
||||
|
||||
@ -35,8 +35,8 @@ To run the tutorial, use this as your configuration file ``config.yaml``.
|
||||
|
||||
.../pypsa-eur % cp config.tutorial.yaml config.yaml
|
||||
|
||||
This configuration is set to download a reduced data set via the rules :mod:`retrieve_databundle`,
|
||||
:mod:`retrieve_natura_raster`, :mod:`retrieve_cutout` totalling at less than 250 MB.
|
||||
This configuration is set to download a reduced data set via the rules :mod:`retrieve_databundle`
|
||||
and :mod:`retrieve_cutout` totalling at less than 250 MB.
|
||||
The full set of data dependencies would consume 5.3 GB.
|
||||
For more information on the data dependencies of PyPSA-Eur, continue reading :ref:`data`.
|
||||
|
||||
|
||||
@ -46,6 +46,7 @@ dependencies:
|
||||
# GIS dependencies:
|
||||
- cartopy
|
||||
- descartes
|
||||
- fiona # explicit for Windows
|
||||
- rasterio<=1.2.9 # 1.2.10 creates error https://github.com/PyPSA/atlite/issues/238
|
||||
|
||||
# PyPSA-Eur-Sec Dependencies
|
||||
|
||||
@ -116,14 +116,19 @@ def nan_statistics(df):
|
||||
keys=['total', 'consecutive', 'max_total_per_month'], axis=1)
|
||||
|
||||
|
||||
def copy_timeslice(load, cntry, start, stop, delta):
|
||||
def copy_timeslice(load, cntry, start, stop, delta, fn_load=None):
|
||||
start = pd.Timestamp(start)
|
||||
stop = pd.Timestamp(stop)
|
||||
if start-delta in load.index and stop in load.index and cntry in load:
|
||||
load.loc[start:stop, cntry] = load.loc[start-delta:stop-delta, cntry].values
|
||||
if (start in load.index and stop in load.index):
|
||||
if start-delta in load.index and stop-delta in load.index and cntry in load:
|
||||
load.loc[start:stop, cntry] = load.loc[start-delta:stop-delta, cntry].values
|
||||
elif fn_load is not None:
|
||||
duration = pd.date_range(freq='h', start=start-delta, end=stop-delta)
|
||||
load_raw = load_timeseries(fn_load, duration, [cntry], powerstatistics)
|
||||
load.loc[start:stop, cntry] = load_raw.loc[start-delta:stop-delta, cntry].values
|
||||
|
||||
|
||||
def manual_adjustment(load, powerstatistics):
|
||||
def manual_adjustment(load, fn_load, powerstatistics):
|
||||
"""
|
||||
Adjust gaps manual for load data from OPSD time-series package.
|
||||
|
||||
@ -150,6 +155,8 @@ def manual_adjustment(load, powerstatistics):
|
||||
powerstatistics: bool
|
||||
Whether argument load comprises the electricity consumption data of
|
||||
the ENTSOE power statistics or of the ENTSOE transparency map
|
||||
load_fn: str
|
||||
File name or url location (file format .csv)
|
||||
|
||||
Returns
|
||||
-------
|
||||
@ -175,7 +182,11 @@ def manual_adjustment(load, powerstatistics):
|
||||
copy_timeslice(load, 'CH', '2010-11-04 04:00', '2010-11-04 22:00', Delta(days=1))
|
||||
copy_timeslice(load, 'NO', '2010-12-09 11:00', '2010-12-09 18:00', Delta(days=1))
|
||||
# whole january missing
|
||||
copy_timeslice(load, 'GB', '2009-12-31 23:00', '2010-01-31 23:00', Delta(days=-364))
|
||||
copy_timeslice(load, 'GB', '2010-01-01 00:00', '2010-01-31 23:00', Delta(days=-365), fn_load)
|
||||
# 1.1. at midnight gets special treatment
|
||||
copy_timeslice(load, 'IE', '2016-01-01 00:00', '2016-01-01 01:00', Delta(days=-366), fn_load)
|
||||
copy_timeslice(load, 'PT', '2016-01-01 00:00', '2016-01-01 01:00', Delta(days=-366), fn_load)
|
||||
copy_timeslice(load, 'GB', '2016-01-01 00:00', '2016-01-01 01:00', Delta(days=-366), fn_load)
|
||||
|
||||
else:
|
||||
if 'ME' in load:
|
||||
@ -206,7 +217,7 @@ if __name__ == "__main__":
|
||||
load = load_timeseries(snakemake.input[0], years, countries, powerstatistics)
|
||||
|
||||
if snakemake.config['load']['manual_adjustments']:
|
||||
load = manual_adjustment(load, powerstatistics)
|
||||
load = manual_adjustment(load, snakemake.input[0], powerstatistics)
|
||||
|
||||
logger.info(f"Linearly interpolate gaps of size {interpolate_limit} and less.")
|
||||
load = load.interpolate(method='linear', limit=interpolate_limit)
|
||||
|
||||
@ -1,89 +0,0 @@
|
||||
# SPDX-FileCopyrightText: : 2017-2020 The PyPSA-Eur Authors
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
"""
|
||||
Rasters the vector data of the `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas onto all cutout regions.
|
||||
|
||||
Relevant Settings
|
||||
-----------------
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
renewable:
|
||||
{technology}:
|
||||
cutout:
|
||||
|
||||
.. seealso::
|
||||
Documentation of the configuration file ``config.yaml`` at
|
||||
:ref:`renewable_cf`
|
||||
|
||||
Inputs
|
||||
------
|
||||
|
||||
- ``data/bundle/natura/Natura2000_end2015.shp``: `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas.
|
||||
|
||||
.. image:: ../img/natura.png
|
||||
:scale: 33 %
|
||||
|
||||
Outputs
|
||||
-------
|
||||
|
||||
- ``resources/natura.tiff``: Rasterized version of `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas to reduce computation times.
|
||||
|
||||
.. image:: ../img/natura.png
|
||||
:scale: 33 %
|
||||
|
||||
Description
|
||||
-----------
|
||||
|
||||
"""
|
||||
|
||||
import logging
|
||||
from _helpers import configure_logging
|
||||
|
||||
import atlite
|
||||
import geopandas as gpd
|
||||
import rasterio as rio
|
||||
from rasterio.features import geometry_mask
|
||||
from rasterio.warp import transform_bounds
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def determine_cutout_xXyY(cutout_name):
|
||||
cutout = atlite.Cutout(cutout_name)
|
||||
assert cutout.crs.to_epsg() == 4326
|
||||
x, X, y, Y = cutout.extent
|
||||
dx, dy = cutout.dx, cutout.dy
|
||||
return [x - dx/2., X + dx/2., y - dy/2., Y + dy/2.]
|
||||
|
||||
|
||||
def get_transform_and_shape(bounds, res):
|
||||
left, bottom = [(b // res)* res for b in bounds[:2]]
|
||||
right, top = [(b // res + 1) * res for b in bounds[2:]]
|
||||
shape = int((top - bottom) // res), int((right - left) / res)
|
||||
transform = rio.Affine(res, 0, left, 0, -res, top)
|
||||
return transform, shape
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if 'snakemake' not in globals():
|
||||
from _helpers import mock_snakemake
|
||||
snakemake = mock_snakemake('build_natura_raster')
|
||||
configure_logging(snakemake)
|
||||
|
||||
cutouts = snakemake.input.cutouts
|
||||
xs, Xs, ys, Ys = zip(*(determine_cutout_xXyY(cutout) for cutout in cutouts))
|
||||
bounds = transform_bounds(4326, 3035, min(xs), min(ys), max(Xs), max(Ys))
|
||||
transform, out_shape = get_transform_and_shape(bounds, res=100)
|
||||
|
||||
# adjusted boundaries
|
||||
shapes = gpd.read_file(snakemake.input.natura).to_crs(3035)
|
||||
raster = ~geometry_mask(shapes.geometry, out_shape[::-1], transform)
|
||||
raster = raster.astype(rio.uint8)
|
||||
|
||||
with rio.open(snakemake.output[0], 'w', driver='GTiff', dtype=rio.uint8,
|
||||
count=1, transform=transform, crs=3035, compress='lzw',
|
||||
width=raster.shape[1], height=raster.shape[0]) as dst:
|
||||
dst.write(raster, indexes=1)
|
||||
@ -227,7 +227,9 @@ if __name__ == '__main__':
|
||||
excluder = atlite.ExclusionContainer(crs=3035, res=100)
|
||||
|
||||
if config['natura']:
|
||||
excluder.add_raster(snakemake.input.natura, nodata=0, allow_no_overlap=True)
|
||||
mask = regions.to_crs(3035).buffer(0) # buffer to avoid invalid geometry
|
||||
natura = gpd.read_file(snakemake.input.natura, mask=mask)
|
||||
excluder.add_geometry(natura.geometry)
|
||||
|
||||
corine = config.get("corine", {})
|
||||
if "grid_codes" in corine:
|
||||
|
||||
@ -102,7 +102,7 @@ def prepare_network(n, solve_opts):
|
||||
|
||||
load_shedding = solve_opts.get('load_shedding')
|
||||
if load_shedding:
|
||||
n.add("Carrier", "Load")
|
||||
n.add("Carrier", "load", color="#dd2e23", nice_name="Load shedding")
|
||||
buses_i = n.buses.query("carrier == 'AC'").index
|
||||
if not np.isscalar(load_shedding): load_shedding = 1e2
|
||||
# intersect between macroeconomic and surveybased
|
||||
|
||||
@ -31,8 +31,6 @@ enable:
|
||||
retrieve_databundle: true
|
||||
build_cutout: false
|
||||
retrieve_cutout: true
|
||||
build_natura_raster: false
|
||||
retrieve_natura_raster: true
|
||||
custom_busmap: false
|
||||
|
||||
electricity:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user