From 5a7c0d7967a57185925d7e6c0df6e7ac70597575 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sun, 20 Mar 2022 10:03:17 +0100 Subject: [PATCH 01/13] remove build/retrieve natura raster, directly use shapefile --- Snakefile | 19 +--- config.default.yaml | 2 - config.tutorial.yaml | 2 - doc/configtables/toplevel.csv | 2 - doc/preparation.rst | 2 - doc/preparation/build_natura_raster.rst | 39 --------- doc/preparation/build_renewable_profiles.rst | 3 - doc/preparation/retrieve.rst | 27 ------ doc/release_notes.rst | 2 + doc/tutorial.rst | 4 +- scripts/build_natura_raster.py | 92 -------------------- scripts/build_renewable_profiles.py | 2 +- test/config.test1.yaml | 2 - 13 files changed, 6 insertions(+), 192 deletions(-) delete mode 100644 doc/preparation/build_natura_raster.rst delete mode 100644 scripts/build_natura_raster.py diff --git a/Snakefile b/Snakefile index 4b8fa0b3..7d2eff1d 100644 --- a/Snakefile +++ b/Snakefile @@ -165,28 +165,11 @@ if config['enable'].get('retrieve_cutout', True): run: move(input[0], output[0]) -if config['enable'].get('build_natura_raster', False): - rule build_natura_raster: - input: - natura="data/bundle/natura/Natura2000_end2015.shp", - cutouts=expand("cutouts/{cutouts}.nc", **config['atlite']) - output: "resources/natura.tiff" - log: "logs/build_natura_raster.log" - script: "scripts/build_natura_raster.py" - - -if config['enable'].get('retrieve_natura_raster', True): - rule retrieve_natura_raster: - input: HTTP.remote("zenodo.org/record/4706686/files/natura.tiff", keep_local=True, static=True) - output: "resources/natura.tiff" - run: move(input[0], output[0]) - - rule build_renewable_profiles: input: base_network="networks/base.nc", corine="data/bundle/corine/g250_clc06_V18_5.tif", - natura="resources/natura.tiff", + natura="data/bundle/natura/Natura2000_end2015.shp", gebco=lambda w: ("data/bundle/GEBCO_2014_2D.nc" if "max_depth" in config["renewable"][w.technology].keys() else []), diff --git a/config.default.yaml b/config.default.yaml index d2bf6159..971293b7 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -33,8 +33,6 @@ enable: retrieve_databundle: true build_cutout: false retrieve_cutout: true - build_natura_raster: false - retrieve_natura_raster: true custom_busmap: false electricity: diff --git a/config.tutorial.yaml b/config.tutorial.yaml index ea624727..67176fdf 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -33,8 +33,6 @@ enable: retrieve_databundle: true build_cutout: false retrieve_cutout: true - build_natura_raster: false - retrieve_natura_raster: true custom_busmap: false electricity: diff --git a/doc/configtables/toplevel.csv b/doc/configtables/toplevel.csv index b7f39d05..8965a0bc 100644 --- a/doc/configtables/toplevel.csv +++ b/doc/configtables/toplevel.csv @@ -12,6 +12,4 @@ enable,,, -- retrieve_databundle,bool,"{true, false}","Switch to retrieve databundle from zenodo via the rule :mod:`retrieve_databundle` or whether to keep a custom databundle located in the corresponding folder." -- build_cutout,bool,"{true, false}","Switch to enable the building of cutouts via the rule :mod:`build_cutout`." -- retrieve_cutout,bool,"{true, false}","Switch to enable the retrieval of cutouts from zenodo with :mod:`retrieve_cutout`." --- build_natura_raster,bool,"{true, false}","Switch to enable the creation of the raster ``natura.tiff`` via the rule :mod:`build_natura_raster`." --- retrieve_natura_raster,bool,"{true, false}","Switch to enable the retrieval of ``natura.tiff`` from zenodo with :mod:`retrieve_natura_raster`." -- custom_busmap,bool,"{true, false}","Switch to enable the use of custom busmaps in rule :mod:`cluster_network`. If activated the rule looks for provided busmaps at ``data/custom_busmap_elec_s{simpl}_{clusters}.csv`` which should have the same format as ``resources/busmap_elec_s{simpl}_{clusters}.csv``, i.e. the index should contain the buses of ``networks/elec_s{simpl}.nc``." diff --git a/doc/preparation.rst b/doc/preparation.rst index dba5e981..7f42190c 100644 --- a/doc/preparation.rst +++ b/doc/preparation.rst @@ -27,7 +27,6 @@ With these and the externally extracted ENTSO-E online map topology Then the process continues by calculating conventional power plant capacities, potentials, and per-unit availability time series for variable renewable energy carriers and hydro power plants with the following rules: - :mod:`build_powerplants` for today's thermal power plant capacities using `powerplantmatching `_ allocating these to the closest substation for each powerplant, -- :mod:`build_natura_raster` for rasterising NATURA2000 natural protection areas, - :mod:`build_renewable_profiles` for the hourly capacity factors and installation potentials constrained by land-use in each substation's Voronoi cell for PV, onshore and offshore wind, and - :mod:`build_hydro_profile` for the hourly per-unit hydro power availability time series. @@ -41,7 +40,6 @@ together into a detailed PyPSA network stored in ``networks/elec.nc``. preparation/build_shapes preparation/build_load_data preparation/build_cutout - preparation/build_natura_raster preparation/prepare_links_p_nom preparation/base_network preparation/build_bus_regions diff --git a/doc/preparation/build_natura_raster.rst b/doc/preparation/build_natura_raster.rst deleted file mode 100644 index e3ec4364..00000000 --- a/doc/preparation/build_natura_raster.rst +++ /dev/null @@ -1,39 +0,0 @@ -.. - SPDX-FileCopyrightText: 2019-2020 The PyPSA-Eur Authors - - SPDX-License-Identifier: CC-BY-4.0 - -.. _natura: - -Rule ``build_natura_raster`` -=============================== - -.. graphviz:: - :align: center - - digraph snakemake_dag { - graph [bgcolor=white, - margin=0, - size="8,5" - ]; - node [fontname=sans, - fontsize=10, - penwidth=2, - shape=box, - style=rounded - ]; - edge [color=grey, - penwidth=2 - ]; - 9 [color="0.22 0.6 0.85", - label=build_renewable_profiles]; - 12 [color="0.31 0.6 0.85", - fillcolor=gray, - label=build_natura_raster, - style=filled]; - 12 -> 9; - } - -| - -.. automodule:: build_natura_raster diff --git a/doc/preparation/build_renewable_profiles.rst b/doc/preparation/build_renewable_profiles.rst index 27e61583..adc4d6ca 100644 --- a/doc/preparation/build_renewable_profiles.rst +++ b/doc/preparation/build_renewable_profiles.rst @@ -41,9 +41,6 @@ Rule ``build_renewable_profiles`` 8 [color="0.00 0.6 0.85", label=build_shapes]; 8 -> 9; - 12 [color="0.31 0.6 0.85", - label=build_natura_raster]; - 12 -> 9; 13 [color="0.56 0.6 0.85", label=build_cutout]; 13 -> 9; diff --git a/doc/preparation/retrieve.rst b/doc/preparation/retrieve.rst index 42479284..21187924 100644 --- a/doc/preparation/retrieve.rst +++ b/doc/preparation/retrieve.rst @@ -50,30 +50,3 @@ The :ref:`tutorial` uses a smaller cutout than required for the full model (30 M .. seealso:: For details see :mod:`build_cutout` and read the `atlite documentation `_. - - -Rule ``retrieve_natura_raster`` -------------------------------- - -.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.4706686.svg - :target: https://doi.org/10.5281/zenodo.4706686 - -This rule, as a substitute for :mod:`build_natura_raster`, downloads an already rasterized version (`natura.tiff `_) of `Natura 2000 `_ natural protection areas to reduce computation times. The file is placed into the ``resources`` sub-directory. - -**Relevant Settings** - -.. code:: yaml - - enable: - build_natura_raster: - -.. seealso:: - Documentation of the configuration file ``config.yaml`` at - :ref:`toplevel_cf` - -**Outputs** - -- ``resources/natura.tiff``: Rasterized version of `Natura 2000 `_ natural protection areas to reduce computation times. - -.. seealso:: - For details see :mod:`build_natura_raster`. diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 3b20fbcf..be1259d9 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -62,6 +62,8 @@ Upcoming Release * New network topology extracted from the ENTSO-E interactive map. +* Remove rules to build or retrieve rasterized NATURA 2000 dataset. Renewable potential calculation now directly uses the shapefiles. + PyPSA-Eur 0.4.0 (22th September 2021) ===================================== diff --git a/doc/tutorial.rst b/doc/tutorial.rst index 17d4e3c1..512683a5 100644 --- a/doc/tutorial.rst +++ b/doc/tutorial.rst @@ -35,8 +35,8 @@ To run the tutorial, use this as your configuration file ``config.yaml``. .../pypsa-eur % cp config.tutorial.yaml config.yaml -This configuration is set to download a reduced data set via the rules :mod:`retrieve_databundle`, -:mod:`retrieve_natura_raster`, :mod:`retrieve_cutout` totalling at less than 250 MB. +This configuration is set to download a reduced data set via the rules :mod:`retrieve_databundle` +and :mod:`retrieve_cutout` totalling at less than 250 MB. The full set of data dependencies would consume 5.3 GB. For more information on the data dependencies of PyPSA-Eur, continue reading :ref:`data`. diff --git a/scripts/build_natura_raster.py b/scripts/build_natura_raster.py deleted file mode 100644 index 71d2c45e..00000000 --- a/scripts/build_natura_raster.py +++ /dev/null @@ -1,92 +0,0 @@ -# SPDX-FileCopyrightText: : 2017-2020 The PyPSA-Eur Authors -# -# SPDX-License-Identifier: MIT - -""" -Rasters the vector data of the `Natura 2000 `_ natural protection areas onto all cutout regions. - -Relevant Settings ------------------ - -.. code:: yaml - - renewable: - {technology}: - cutout: - -.. seealso:: - Documentation of the configuration file ``config.yaml`` at - :ref:`renewable_cf` - -Inputs ------- - -- ``data/bundle/natura/Natura2000_end2015.shp``: `Natura 2000 `_ natural protection areas. - - .. image:: ../img/natura.png - :scale: 33 % - -Outputs -------- - -- ``resources/natura.tiff``: Rasterized version of `Natura 2000 `_ natural protection areas to reduce computation times. - - .. image:: ../img/natura.png - :scale: 33 % - -Description ------------ - -""" - -import logging -from _helpers import configure_logging, retrieve_snakemake_keys - -import atlite -import geopandas as gpd -import rasterio as rio -from rasterio.features import geometry_mask -from rasterio.warp import transform_bounds - -logger = logging.getLogger(__name__) - - -def determine_cutout_xXyY(cutout_name): - cutout = atlite.Cutout(cutout_name) - assert cutout.crs.to_epsg() == 4326 - x, X, y, Y = cutout.extent - dx, dy = cutout.dx, cutout.dy - return [x - dx/2., X + dx/2., y - dy/2., Y + dy/2.] - - -def get_transform_and_shape(bounds, res): - left, bottom = [(b // res)* res for b in bounds[:2]] - right, top = [(b // res + 1) * res for b in bounds[2:]] - shape = int((top - bottom) // res), int((right - left) / res) - transform = rio.Affine(res, 0, left, 0, -res, top) - return transform, shape - - -if __name__ == "__main__": - if 'snakemake' not in globals(): - from _helpers import mock_snakemake - snakemake = mock_snakemake('build_natura_raster') - configure_logging(snakemake) - - paths, config, wildcards, logs, out = retrieve_snakemake_keys(snakemake) - - cutouts = paths.cutouts - xs, Xs, ys, Ys = zip(*(determine_cutout_xXyY(cutout) for cutout in cutouts)) - bounds = transform_bounds(4326, 3035, min(xs), min(ys), max(Xs), max(Ys)) - transform, out_shape = get_transform_and_shape(bounds, res=100) - - # adjusted boundaries - shapes = gpd.read_file(paths.natura).to_crs(3035) - raster = ~geometry_mask(shapes.geometry, out_shape[::-1], transform) - raster = raster.astype(rio.uint8) - - with rio.open(out[0], 'w', driver='GTiff', dtype=rio.uint8, - count=1, transform=transform, crs=3035, compress='lzw', - width=raster.shape[1], height=raster.shape[0]) as dst: - dst.write(raster, indexes=1) - diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 36845da5..41d208c1 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -224,7 +224,7 @@ if __name__ == '__main__': excluder = atlite.ExclusionContainer(crs=3035, res=100) if config['natura']: - excluder.add_raster(snakemake.input.natura, nodata=0, allow_no_overlap=True) + excluder.add_geometry(snakemake.input.natura) corine = config.get("corine", {}) if "grid_codes" in corine: diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 2986037b..d017cafc 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -32,8 +32,6 @@ enable: retrieve_databundle: true build_cutout: false retrieve_cutout: true - build_natura_raster: false - retrieve_natura_raster: true custom_busmap: false electricity: From 7ba1f26eb8c2d87d41b34c0f9f805550639ca6d8 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sun, 20 Mar 2022 20:56:38 +0100 Subject: [PATCH 02/13] must read NATURA shape before add_geometry --- scripts/build_renewable_profiles.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 41d208c1..2b95e669 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -224,7 +224,9 @@ if __name__ == '__main__': excluder = atlite.ExclusionContainer(crs=3035, res=100) if config['natura']: - excluder.add_geometry(snakemake.input.natura) + mask = regions.to_crs(3035).buffer(0) # buffer to avoid invalid geometry + natura = gpd.read_file(snakemake.input.natura, mask=mask) + excluder.add_geometry(natura.geometry) corine = config.get("corine", {}) if "grid_codes" in corine: From 31f9864daf45fda94102ec197227b4297981cb3b Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sun, 20 Mar 2022 21:01:07 +0100 Subject: [PATCH 03/13] omit NATURA from testing and tutorial due to file size# --- Snakefile | 4 +++- config.tutorial.yaml | 8 ++++---- test/config.test1.yaml | 8 ++++---- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Snakefile b/Snakefile index 7d2eff1d..50e0d2f8 100644 --- a/Snakefile +++ b/Snakefile @@ -169,7 +169,9 @@ rule build_renewable_profiles: input: base_network="networks/base.nc", corine="data/bundle/corine/g250_clc06_V18_5.tif", - natura="data/bundle/natura/Natura2000_end2015.shp", + natura=lambda w: ("data/bundle/natura/Natura2000_end2015.shp" + if config["renewable"][w.technology]["natura"] + else []), gebco=lambda w: ("data/bundle/GEBCO_2014_2D.nc" if "max_depth" in config["renewable"][w.technology].keys() else []), diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 67176fdf..da3e10cc 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -77,7 +77,7 @@ renewable: 24, 25, 26, 27, 28, 29, 31, 32] distance: 1000 distance_grid_codes: [1, 2, 3, 4, 5, 6] - natura: true + natura: false potential: simple # or conservative clip_p_max_pu: 1.e-2 offwind-ac: @@ -88,7 +88,7 @@ renewable: capacity_per_sqkm: 3 # correction_factor: 0.93 corine: [44, 255] - natura: true + natura: false max_shore_distance: 30000 potential: simple # or conservative clip_p_max_pu: 1.e-2 @@ -101,7 +101,7 @@ renewable: capacity_per_sqkm: 3 # correction_factor: 0.93 corine: [44, 255] - natura: true + natura: false min_shore_distance: 30000 potential: simple # or conservative clip_p_max_pu: 1.e-2 @@ -123,7 +123,7 @@ renewable: # correction_factor: 0.854337 corine: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 26, 31, 32] - natura: true + natura: false potential: simple # or conservative clip_p_max_pu: 1.e-2 diff --git a/test/config.test1.yaml b/test/config.test1.yaml index d017cafc..7a48ea87 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -76,7 +76,7 @@ renewable: 24, 25, 26, 27, 28, 29, 31, 32] distance: 1000 distance_grid_codes: [1, 2, 3, 4, 5, 6] - natura: true + natura: false potential: simple # or conservative clip_p_max_pu: 1.e-2 offwind-ac: @@ -87,7 +87,7 @@ renewable: capacity_per_sqkm: 3 # correction_factor: 0.93 corine: [44, 255] - natura: true + natura: false max_shore_distance: 30000 potential: simple # or conservative clip_p_max_pu: 1.e-2 @@ -100,7 +100,7 @@ renewable: capacity_per_sqkm: 3 # correction_factor: 0.93 corine: [44, 255] - natura: true + natura: false min_shore_distance: 30000 potential: simple # or conservative clip_p_max_pu: 1.e-2 @@ -121,7 +121,7 @@ renewable: correction_factor: 0.854337 corine: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 26, 31, 32] - natura: true + natura: false potential: simple # or conservative clip_p_max_pu: 1.e-2 From ad4bd56ab27be0c16620f80ecfb9a0dcf132a29c Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 24 Mar 2022 12:36:54 +0100 Subject: [PATCH 04/13] cache data bundle --- .github/workflows/ci.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c753deab..0406e742 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,7 +19,8 @@ on: - cron: "0 5 * * TUE" env: - CACHE_NUMBER: 1 # Change this value to manually reset the environment cache + CONDA_CACHE_NUMBER: 1 # Change this value to manually reset the environment cache + BUNDLE_CACHE_NUMBER: 1 jobs: build: @@ -66,6 +67,12 @@ jobs: miniforge-version: latest activate-environment: pypsa-eur use-mamba: true + + - name: Cache data bundle + uses: actions/cache@v3 + with: + path: data/bundle + key: data-bundle-${{ env.BUNDLE_CACHE_NUMBER }} - name: Set cache date run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV @@ -75,7 +82,7 @@ jobs: id: cache with: path: ${{ matrix.prefix }} - key: ${{ matrix.label }}-conda-${{ hashFiles('envs/environment.yaml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }} + key: ${{ matrix.label }}-conda-${{ hashFiles('envs/environment.yaml') }}-${{ env.DATE }}-${{ env.CONDA_CACHE_NUMBER }} - name: Update environment due to outdated or unavailable cache run: mamba env update -n pypsa-eur -f envs/environment.yaml From d24b9a48832e48499464acfdcbe2a6a2e66c0765 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 24 Mar 2022 12:40:29 +0100 Subject: [PATCH 05/13] run CI on PRs targeting non-master branches --- .github/workflows/ci.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0406e742..0421ef82 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,8 +13,6 @@ on: branches: - master pull_request: - branches: - - master schedule: - cron: "0 5 * * TUE" @@ -73,7 +71,7 @@ jobs: with: path: data/bundle key: data-bundle-${{ env.BUNDLE_CACHE_NUMBER }} - + - name: Set cache date run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV From f90bb92e1a4199ab1d76e0a4645d53a076f84bfa Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 24 Mar 2022 13:37:21 +0100 Subject: [PATCH 06/13] automatically download NATURA2000 gpkg and use it --- Snakefile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index 50e0d2f8..a28bd4fd 100644 --- a/Snakefile +++ b/Snakefile @@ -67,6 +67,12 @@ if config['enable'].get('retrieve_databundle', True): script: 'scripts/retrieve_databundle.py' +rule retrieve_natura_data: + input: HTTP.remote("sdi.eea.europa.eu/datashare/s/H6QGCybMdLLnywo/download?path=%2FNatura2000_end2020_gpkg&files=Natura2000_end2020.gpkg", keep_local=True, static=True) + output: "data/Natura2000_end2020.gpkg" + run: move(input[0], output[0]) + + rule retrieve_load_data: input: HTTP.remote("data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv", keep_local=True, static=True) output: "data/load_raw.csv" @@ -169,7 +175,7 @@ rule build_renewable_profiles: input: base_network="networks/base.nc", corine="data/bundle/corine/g250_clc06_V18_5.tif", - natura=lambda w: ("data/bundle/natura/Natura2000_end2015.shp" + natura=lambda w: ("data/Natura2000_end2020.gpkg" if config["renewable"][w.technology]["natura"] else []), gebco=lambda w: ("data/bundle/GEBCO_2014_2D.nc" From 273ac5a1921c623c509b30f44885af466b84bbff Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 24 Mar 2022 14:07:44 +0100 Subject: [PATCH 07/13] cache full data folder and cutouts --- .github/workflows/ci.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0421ef82..98c024b6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,7 +18,7 @@ on: env: CONDA_CACHE_NUMBER: 1 # Change this value to manually reset the environment cache - BUNDLE_CACHE_NUMBER: 1 + DATA_CACHE_NUMBER: 1 jobs: build: @@ -66,11 +66,13 @@ jobs: activate-environment: pypsa-eur use-mamba: true - - name: Cache data bundle + - name: Cache data and cutouts folders uses: actions/cache@v3 with: - path: data/bundle - key: data-bundle-${{ env.BUNDLE_CACHE_NUMBER }} + path: | + data + cutouts + key: data-cutouts-${{ env.DATA_CACHE_NUMBER }} - name: Set cache date run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV From 9f4e61d822c9c18600d6f09ad53d8e4a9159e213 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 24 Mar 2022 14:13:37 +0100 Subject: [PATCH 08/13] refresh data/cutouts folders cache every week --- .github/workflows/ci.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 98c024b6..af285b7a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -72,10 +72,12 @@ jobs: path: | data cutouts - key: data-cutouts-${{ env.DATA_CACHE_NUMBER }} + key: data-cutouts-${{ env.WEEK }}-${{ env.DATA_CACHE_NUMBER }} - - name: Set cache date - run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV + - name: Set cache dates + run: | + echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV + echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV - name: Create environment cache uses: actions/cache@v2 From 5833ac90eb186fa11eb85ba7766e35a39438f141 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 24 Mar 2022 14:14:33 +0100 Subject: [PATCH 09/13] move up cache date setting --- .github/workflows/ci.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index af285b7a..d447a56f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -66,6 +66,11 @@ jobs: activate-environment: pypsa-eur use-mamba: true + - name: Set cache dates + run: | + echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV + echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV + - name: Cache data and cutouts folders uses: actions/cache@v3 with: @@ -74,11 +79,6 @@ jobs: cutouts key: data-cutouts-${{ env.WEEK }}-${{ env.DATA_CACHE_NUMBER }} - - name: Set cache dates - run: | - echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV - echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV - - name: Create environment cache uses: actions/cache@v2 id: cache From 62e7653d7edff3e4cfa554008b62bad0852857e6 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 24 Mar 2022 14:37:18 +0100 Subject: [PATCH 10/13] add release notes --- doc/release_notes.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index be1259d9..acc18851 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -64,6 +64,9 @@ Upcoming Release * Remove rules to build or retrieve rasterized NATURA 2000 dataset. Renewable potential calculation now directly uses the shapefiles. +* Cache data and cutouts folders. This cache will be updated weekly. + +* Add rule to automatically retrieve Natura2000 natural protection areas. Switch of file format to GPKG. PyPSA-Eur 0.4.0 (22th September 2021) ===================================== From da8a7f1293c2be282a329073f51dad689b77c8e4 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 24 Mar 2022 14:39:36 +0100 Subject: [PATCH 11/13] consider again NATURA in testing --- test/config.test1.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 7a48ea87..d017cafc 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -76,7 +76,7 @@ renewable: 24, 25, 26, 27, 28, 29, 31, 32] distance: 1000 distance_grid_codes: [1, 2, 3, 4, 5, 6] - natura: false + natura: true potential: simple # or conservative clip_p_max_pu: 1.e-2 offwind-ac: @@ -87,7 +87,7 @@ renewable: capacity_per_sqkm: 3 # correction_factor: 0.93 corine: [44, 255] - natura: false + natura: true max_shore_distance: 30000 potential: simple # or conservative clip_p_max_pu: 1.e-2 @@ -100,7 +100,7 @@ renewable: capacity_per_sqkm: 3 # correction_factor: 0.93 corine: [44, 255] - natura: false + natura: true min_shore_distance: 30000 potential: simple # or conservative clip_p_max_pu: 1.e-2 @@ -121,7 +121,7 @@ renewable: correction_factor: 0.854337 corine: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 26, 31, 32] - natura: false + natura: true potential: simple # or conservative clip_p_max_pu: 1.e-2 From 832cc8e5e8ddb8c9cb17d3f53f5493aa28abad10 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 28 Mar 2022 10:56:52 +0200 Subject: [PATCH 12/13] Windows compatibility through request string --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index 16d1c88e..53c4f6c9 100644 --- a/Snakefile +++ b/Snakefile @@ -68,7 +68,7 @@ if config['enable'].get('retrieve_databundle', True): rule retrieve_natura_data: - input: HTTP.remote("sdi.eea.europa.eu/datashare/s/H6QGCybMdLLnywo/download?path=%2FNatura2000_end2020_gpkg&files=Natura2000_end2020.gpkg", keep_local=True, static=True) + input: HTTP.remote("sdi.eea.europa.eu/datashare/s/H6QGCybMdLLnywo/download", additional_request_string="?path=%2FNatura2000_end2020_gpkg&files=Natura2000_end2020.gpkg", static=True) output: "data/Natura2000_end2020.gpkg" run: move(input[0], output[0]) From d3589be761c26a50484f3c3ad95591f2b6eb7957 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 28 Mar 2022 13:17:21 +0200 Subject: [PATCH 13/13] test explicit fiona for Windows --- envs/environment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/envs/environment.yaml b/envs/environment.yaml index 0c881720..eca18332 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -45,6 +45,7 @@ dependencies: - cartopy - descartes - rasterio + - fiona # explicit for Windows # PyPSA-Eur-Sec Dependencies - geopy