Merge branch 'master' into landlock-fix-alterative

.github/workflows/ci.yaml

@@ -13,13 +13,12 @@ on:
     branches:
     - master
   pull_request:
-    branches:
-    - master
   schedule:
     - cron: "0 5 * * TUE"
 
 env:
-  CACHE_NUMBER: 1 # Change this value to manually reset the environment cache
+  CONDA_CACHE_NUMBER: 1 # Change this value to manually reset the environment cache
+  DATA_CACHE_NUMBER: 1
 
 jobs:
   build:
@@ -66,16 +65,26 @@ jobs:
             miniforge-version: latest
             activate-environment: pypsa-eur
             use-mamba: true
-            
+
-      - name: Set cache date
+      - name: Set cache dates
-        run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV
+        run: |
+          echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV
+          echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV
+
+      - name: Cache data and cutouts folders
+        uses: actions/cache@v3
+        with:
+          path: |
+            data
+            cutouts
+          key: data-cutouts-${{ env.WEEK }}-${{ env.DATA_CACHE_NUMBER }}
 
       - name: Create environment cache
         uses: actions/cache@v2
         id: cache
         with:
           path: ${{ matrix.prefix }}
-          key: ${{ matrix.label }}-conda-${{ hashFiles('envs/environment.yaml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
+          key: ${{ matrix.label }}-conda-${{ hashFiles('envs/environment.yaml') }}-${{ env.DATE }}-${{ env.CONDA_CACHE_NUMBER }}
 
       - name: Update environment due to outdated or unavailable cache
         run: mamba env update -n pypsa-eur -f envs/environment.yaml

Snakefile

@@ -67,6 +67,12 @@ if config['enable'].get('retrieve_databundle', True):
         script: 'scripts/retrieve_databundle.py'
 
 
+rule retrieve_natura_data:
+    input: HTTP.remote("sdi.eea.europa.eu/datashare/s/H6QGCybMdLLnywo/download", additional_request_string="?path=%2FNatura2000_end2020_gpkg&files=Natura2000_end2020.gpkg", static=True)
+    output: "data/Natura2000_end2020.gpkg"
+    run: move(input[0], output[0])
+
+
 rule retrieve_load_data:
     input: HTTP.remote("data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv", keep_local=True, static=True)
     output: "data/load_raw.csv"
@@ -165,28 +171,13 @@ if config['enable'].get('retrieve_cutout', True):
         run: move(input[0], output[0])
 
 
-if config['enable'].get('build_natura_raster', False):
-    rule build_natura_raster:
-        input:
-            natura="data/bundle/natura/Natura2000_end2015.shp",
-            cutouts=expand("cutouts/{cutouts}.nc", **config['atlite'])
-        output: "resources/natura.tiff"
-        log: "logs/build_natura_raster.log"
-        script: "scripts/build_natura_raster.py"
-
-
-if config['enable'].get('retrieve_natura_raster', True):
-    rule retrieve_natura_raster:
-        input: HTTP.remote("zenodo.org/record/4706686/files/natura.tiff", keep_local=True, static=True)
-        output: "resources/natura.tiff"
-        run: move(input[0], output[0])
-
-
 rule build_renewable_profiles:
     input:
         base_network="networks/base.nc",
         corine="data/bundle/corine/g250_clc06_V18_5.tif",
-        natura="resources/natura.tiff",
+        natura=lambda w: ("data/Natura2000_end2020.gpkg"
+                          if config["renewable"][w.technology]["natura"]
+                          else []),
         gebco=lambda w: ("data/bundle/GEBCO_2014_2D.nc"
                          if "max_depth" in config["renewable"][w.technology].keys()
                          else []),

config.default.yaml

@@ -33,8 +33,6 @@ enable:
   retrieve_databundle: true
   build_cutout: false
   retrieve_cutout: true
-  build_natura_raster: false
-  retrieve_natura_raster: true
   custom_busmap: false
 
 electricity:

config.tutorial.yaml

@@ -33,8 +33,6 @@ enable:
   retrieve_databundle: true
   build_cutout: false
   retrieve_cutout: true
-  build_natura_raster: false
-  retrieve_natura_raster: true
   custom_busmap: false
 
 electricity:
@@ -79,7 +77,7 @@ renewable:
                    24, 25, 26, 27, 28, 29, 31, 32]
       distance: 1000
       distance_grid_codes: [1, 2, 3, 4, 5, 6]
-    natura: true
+    natura: false
     potential: simple # or conservative
     clip_p_max_pu: 1.e-2
   offwind-ac:
@@ -90,7 +88,7 @@ renewable:
     capacity_per_sqkm: 3
     # correction_factor: 0.93
     corine: [44, 255]
-    natura: true
+    natura: false
     max_shore_distance: 30000
     potential: simple # or conservative
     clip_p_max_pu: 1.e-2
@@ -103,7 +101,7 @@ renewable:
     capacity_per_sqkm: 3
     # correction_factor: 0.93
     corine: [44, 255]
-    natura: true
+    natura: false
     min_shore_distance: 30000
     potential: simple # or conservative
     clip_p_max_pu: 1.e-2
@@ -125,7 +123,7 @@ renewable:
     # correction_factor: 0.854337
     corine: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
              14, 15, 16, 17, 18, 19, 20, 26, 31, 32]
-    natura: true
+    natura: false
     potential: simple # or conservative
     clip_p_max_pu: 1.e-2
 

doc/configtables/toplevel.csv

@@ -12,6 +12,4 @@ enable,,,
 -- retrieve_databundle,bool,"{true, false}","Switch to retrieve databundle from zenodo via the rule :mod:`retrieve_databundle` or whether to keep a custom databundle located in the corresponding folder."
 -- build_cutout,bool,"{true, false}","Switch to enable the building of cutouts via the rule :mod:`build_cutout`."
 -- retrieve_cutout,bool,"{true, false}","Switch to enable the retrieval of cutouts from zenodo with :mod:`retrieve_cutout`."
--- build_natura_raster,bool,"{true, false}","Switch to enable the creation of the raster ``natura.tiff`` via the rule :mod:`build_natura_raster`."
--- retrieve_natura_raster,bool,"{true, false}","Switch to enable the retrieval of ``natura.tiff`` from zenodo with :mod:`retrieve_natura_raster`."
 -- custom_busmap,bool,"{true, false}","Switch to enable the use of custom busmaps in rule :mod:`cluster_network`. If activated the rule looks for provided busmaps at ``data/custom_busmap_elec_s{simpl}_{clusters}.csv`` which should have the same format as ``resources/busmap_elec_s{simpl}_{clusters}.csv``, i.e. the index should contain the buses of ``networks/elec_s{simpl}.nc``."

doc/preparation.rst

@@ -27,7 +27,6 @@ With these and the externally extracted ENTSO-E online map topology
 Then the process continues by calculating conventional power plant capacities, potentials, and per-unit availability time series for variable renewable energy carriers and hydro power plants with the following rules:
 
 - :mod:`build_powerplants` for today's thermal power plant capacities using `powerplantmatching <https://github.com/FRESNA/powerplantmatching>`_ allocating these to the closest substation for each powerplant,
-- :mod:`build_natura_raster` for rasterising NATURA2000 natural protection areas,
 - :mod:`build_renewable_profiles` for the hourly capacity factors and installation potentials constrained by land-use in each substation's Voronoi cell for PV, onshore and offshore wind, and
 - :mod:`build_hydro_profile` for the hourly per-unit hydro power availability time series.
 
@@ -41,7 +40,6 @@ together into a detailed PyPSA network stored in ``networks/elec.nc``.
    preparation/build_shapes
    preparation/build_load_data
    preparation/build_cutout
-   preparation/build_natura_raster
    preparation/prepare_links_p_nom
    preparation/base_network
    preparation/build_bus_regions

doc/preparation/build_natura_raster.rst

@@ -1,39 +0,0 @@
-..
-  SPDX-FileCopyrightText: 2019-2020 The PyPSA-Eur Authors
-
-  SPDX-License-Identifier: CC-BY-4.0
-
-.. _natura:
-
-Rule ``build_natura_raster``
-===============================
-
-.. graphviz::
-    :align: center
-
-    digraph snakemake_dag {
-        graph [bgcolor=white,
-            margin=0,
-            size="8,5"
-        ];
-        node [fontname=sans,
-            fontsize=10,
-            penwidth=2,
-            shape=box,
-            style=rounded
-        ];
-        edge [color=grey,
-            penwidth=2
-        ];
-        9	 [color="0.22 0.6 0.85",
-            label=build_renewable_profiles];
-        12	 [color="0.31 0.6 0.85",
-            fillcolor=gray,
-            label=build_natura_raster,
-            style=filled];
-        12 -> 9;
-    }
-
-|
-
-.. automodule:: build_natura_raster

doc/preparation/build_renewable_profiles.rst

@@ -41,9 +41,6 @@ Rule ``build_renewable_profiles``
         8	 [color="0.00 0.6 0.85",
             label=build_shapes];
         8 -> 9;
-        12	 [color="0.31 0.6 0.85",
-            label=build_natura_raster];
-        12 -> 9;
         13	 [color="0.56 0.6 0.85",
             label=build_cutout];
         13 -> 9;

doc/preparation/retrieve.rst

@@ -50,30 +50,3 @@ The :ref:`tutorial` uses a smaller cutout than required for the full model (30 M
 
 .. seealso::
     For details see :mod:`build_cutout` and read the `atlite documentation <https://atlite.readthedocs.io>`_.
-
-
-Rule ``retrieve_natura_raster``
--------------------------------
-
-.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.4706686.svg
-   :target: https://doi.org/10.5281/zenodo.4706686
-
-This rule, as a substitute for :mod:`build_natura_raster`, downloads an already rasterized version (`natura.tiff <https://zenodo.org/record/4706686/files/natura.tiff>`_) of `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas to reduce computation times. The file is placed into the ``resources`` sub-directory.
-
-**Relevant Settings**
-
-.. code:: yaml
-
-    enable:
-        build_natura_raster:
-
-.. seealso::
-    Documentation of the configuration file ``config.yaml`` at
-    :ref:`toplevel_cf`
-
-**Outputs**
-
-- ``resources/natura.tiff``: Rasterized version of `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas to reduce computation times.
-
-.. seealso::
-    For details see :mod:`build_natura_raster`.

doc/release_notes.rst

@@ -64,6 +64,11 @@ Upcoming Release
 
 * New network topology extracted from the ENTSO-E interactive map.
 
+* Remove rules to build or retrieve rasterized NATURA 2000 dataset. Renewable potential calculation now directly uses the shapefiles.
+
+* Cache data and cutouts folders. This cache will be updated weekly.
+
+* Add rule to automatically retrieve Natura2000 natural protection areas. Switch of file format to GPKG.
 * The unused argument ``simple_hvdc_costs`` in :mod:`add_electricity` was removed.
 
 * Iterative solving with impedance updates is skipped if there are no expandable lines.

doc/tutorial.rst

@@ -35,8 +35,8 @@ To run the tutorial, use this as your configuration file ``config.yaml``.
 
     .../pypsa-eur % cp config.tutorial.yaml config.yaml
 
-This configuration is set to download a reduced data set via the rules :mod:`retrieve_databundle`,
+This configuration is set to download a reduced data set via the rules :mod:`retrieve_databundle`
-:mod:`retrieve_natura_raster`, :mod:`retrieve_cutout` totalling at less than 250 MB.
+and :mod:`retrieve_cutout` totalling at less than 250 MB.
 The full set of data dependencies would consume 5.3 GB.
 For more information on the data dependencies of PyPSA-Eur, continue reading :ref:`data`.
 

envs/environment.yaml

@@ -45,6 +45,7 @@ dependencies:
   # GIS dependencies:
   - cartopy
   - descartes
+  - fiona # explicit for Windows
   - rasterio<=1.2.9  # 1.2.10 creates error https://github.com/PyPSA/atlite/issues/238
 
   # PyPSA-Eur-Sec Dependencies

scripts/build_load_data.py

@@ -116,14 +116,19 @@ def nan_statistics(df):
                  keys=['total', 'consecutive', 'max_total_per_month'], axis=1)
 
 
-def copy_timeslice(load, cntry, start, stop, delta):
+def copy_timeslice(load, cntry, start, stop, delta, fn_load=None):
     start = pd.Timestamp(start)
     stop = pd.Timestamp(stop)
-    if start-delta in load.index and stop in load.index and cntry in load:
+    if (start in load.index and stop in load.index):
-        load.loc[start:stop, cntry] = load.loc[start-delta:stop-delta, cntry].values
+        if start-delta in load.index and stop-delta in load.index and cntry in load:
+            load.loc[start:stop, cntry] = load.loc[start-delta:stop-delta, cntry].values
+        elif fn_load is not None:
+            duration = pd.date_range(freq='h', start=start-delta, end=stop-delta)
+            load_raw = load_timeseries(fn_load, duration, [cntry], powerstatistics)
+            load.loc[start:stop, cntry] = load_raw.loc[start-delta:stop-delta, cntry].values
 
 
-def manual_adjustment(load, powerstatistics):
+def manual_adjustment(load, fn_load, powerstatistics):
     """
     Adjust gaps manual for load data from OPSD time-series package.
 
@@ -150,6 +155,8 @@ def manual_adjustment(load, powerstatistics):
     powerstatistics: bool
         Whether argument load comprises the electricity consumption data of
         the ENTSOE power statistics or of the ENTSOE transparency map
+   load_fn: str
+        File name or url location (file format .csv)
 
     Returns
     -------
@@ -175,7 +182,11 @@ def manual_adjustment(load, powerstatistics):
         copy_timeslice(load, 'CH', '2010-11-04 04:00', '2010-11-04 22:00', Delta(days=1))
         copy_timeslice(load, 'NO', '2010-12-09 11:00', '2010-12-09 18:00', Delta(days=1))
         # whole january missing
-        copy_timeslice(load, 'GB', '2009-12-31 23:00', '2010-01-31 23:00', Delta(days=-364))
+        copy_timeslice(load, 'GB', '2010-01-01 00:00', '2010-01-31 23:00', Delta(days=-365), fn_load)
+        # 1.1. at midnight gets special treatment
+        copy_timeslice(load, 'IE', '2016-01-01 00:00', '2016-01-01 01:00', Delta(days=-366), fn_load)
+        copy_timeslice(load, 'PT', '2016-01-01 00:00', '2016-01-01 01:00', Delta(days=-366), fn_load)
+        copy_timeslice(load, 'GB', '2016-01-01 00:00', '2016-01-01 01:00', Delta(days=-366), fn_load)
 
     else:
         if 'ME' in load:
@@ -206,7 +217,7 @@ if __name__ == "__main__":
     load = load_timeseries(snakemake.input[0], years, countries, powerstatistics)
 
     if snakemake.config['load']['manual_adjustments']:
-        load = manual_adjustment(load, powerstatistics)
+        load = manual_adjustment(load, snakemake.input[0], powerstatistics)
 
     logger.info(f"Linearly interpolate gaps of size {interpolate_limit} and less.")
     load = load.interpolate(method='linear', limit=interpolate_limit)

scripts/build_natura_raster.py

@@ -1,89 +0,0 @@
-# SPDX-FileCopyrightText: : 2017-2020 The PyPSA-Eur Authors
-#
-# SPDX-License-Identifier: MIT
-
-"""
-Rasters the vector data of the `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas onto all cutout regions.
-
-Relevant Settings
------------------
-
-.. code:: yaml
-
-    renewable:
-        {technology}:
-            cutout:
-
-.. seealso::
-    Documentation of the configuration file ``config.yaml`` at
-    :ref:`renewable_cf`
-
-Inputs
-------
-
-- ``data/bundle/natura/Natura2000_end2015.shp``: `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas.
-
-    .. image:: ../img/natura.png
-        :scale: 33 %
-
-Outputs
--------
-
-- ``resources/natura.tiff``: Rasterized version of `Natura 2000 <https://en.wikipedia.org/wiki/Natura_2000>`_ natural protection areas to reduce computation times.
-
-    .. image:: ../img/natura.png
-        :scale: 33 %
-
-Description
------------
-
-"""
-
-import logging
-from _helpers import configure_logging
-
-import atlite
-import geopandas as gpd
-import rasterio as rio
-from rasterio.features import geometry_mask
-from rasterio.warp import transform_bounds
-
-logger = logging.getLogger(__name__)
-
-
-def determine_cutout_xXyY(cutout_name):
-    cutout = atlite.Cutout(cutout_name)
-    assert cutout.crs.to_epsg() == 4326
-    x, X, y, Y = cutout.extent
-    dx, dy = cutout.dx, cutout.dy
-    return [x - dx/2., X + dx/2., y - dy/2., Y + dy/2.]
-
-
-def get_transform_and_shape(bounds, res):
-    left, bottom = [(b // res)* res for b in bounds[:2]]
-    right, top = [(b // res + 1) * res for b in bounds[2:]]
-    shape = int((top - bottom) // res), int((right - left) / res)
-    transform = rio.Affine(res, 0, left, 0, -res, top)
-    return transform, shape
-
-
-if __name__ == "__main__":
-    if 'snakemake' not in globals():
-        from _helpers import mock_snakemake
-        snakemake = mock_snakemake('build_natura_raster')
-    configure_logging(snakemake)
-
-    cutouts = snakemake.input.cutouts
-    xs, Xs, ys, Ys = zip(*(determine_cutout_xXyY(cutout) for cutout in cutouts))
-    bounds = transform_bounds(4326, 3035, min(xs), min(ys), max(Xs), max(Ys))
-    transform, out_shape = get_transform_and_shape(bounds, res=100)
-
-    # adjusted boundaries
-    shapes = gpd.read_file(snakemake.input.natura).to_crs(3035)
-    raster = ~geometry_mask(shapes.geometry, out_shape[::-1], transform)
-    raster = raster.astype(rio.uint8)
-
-    with rio.open(snakemake.output[0], 'w', driver='GTiff', dtype=rio.uint8,
-                  count=1, transform=transform, crs=3035, compress='lzw',
-                  width=raster.shape[1], height=raster.shape[0]) as dst:
-        dst.write(raster, indexes=1)

scripts/build_renewable_profiles.py

@@ -231,7 +231,9 @@ if __name__ == '__main__':
     excluder = atlite.ExclusionContainer(crs=3035, res=100)
 
     if config['natura']:
-        excluder.add_raster(snakemake.input.natura, nodata=0, allow_no_overlap=True)
+        mask = regions.to_crs(3035).buffer(0) # buffer to avoid invalid geometry
+        natura = gpd.read_file(snakemake.input.natura, mask=mask)
+        excluder.add_geometry(natura.geometry)
 
     corine = config.get("corine", {})
     if "grid_codes" in corine:

scripts/solve_network.py

@@ -100,7 +100,7 @@ def prepare_network(n, solve_opts):
             df.where(df>solve_opts['clip_p_max_pu'], other=0., inplace=True)
 
     if solve_opts.get('load_shedding'):
-        n.add("Carrier", "Load")
+        n.add("Carrier", "load", color="#dd2e23", nice_name="Load shedding")
         buses_i = n.buses.query("carrier == 'AC'").index
         n.madd("Generator", buses_i, " load",
                bus=buses_i,

test/config.test1.yaml

@@ -32,8 +32,6 @@ enable:
   retrieve_databundle: true
   build_cutout: false
   retrieve_cutout: true
-  build_natura_raster: false
-  retrieve_natura_raster: true
   custom_busmap: false
 
 electricity: