Merge pull request #924 from PyPSA/merged-electricity-demand

Merged electricity demand sources (powerstatistics, OPSD)
2024-02-12 10:56:13 +01:00 · 2024-02-12 10:56:13 +01:00 · e8c8d72b3f
commit e8c8d72b3f
parent cbb3ab36a6 09ada56bf1
8 changed files with 139 additions and 137 deletions
--- a/doc/configtables/load.csv
+++ b/doc/configtables/load.csv
@ -1,5 +1,4 @@
 ,Unit,Values,Description
-power_statistics,bool,"{true, false}",Whether to load the electricity consumption data of the ENTSOE power statistics (only for files from 2019 and before) or from the ENTSOE transparency data (only has load data from 2015 onwards).
 interpolate_limit,hours,integer,"Maximum gap size (consecutive nans) which interpolated linearly."
 time_shift_for_large_gaps,string,string,"Periods which are used for copying time-slices in order to fill large gaps of nans. Have to be valid ``pandas`` period strings."
 manual_adjustments,bool,"{true, false}","Whether to adjust the load data manually according to the function in :func:`manual_adjustment`."
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@ -10,6 +10,9 @@ Release Notes
 Upcoming Release
 ================

+* Merged two OPSD time series data versions into such that the option ``load:
+  power_statistics:`` becomes superfluous and was hence removed.
+
 * Add new default to overdimension heating in individual buildings. This allows
  them to cover heat demand peaks e.g. 10% higher than those in the data. The
  disadvantage of manipulating the costs is that the capacity is then not quite
--- a/doc/retrieve.rst
+++ b/doc/retrieve.rst
@ -91,7 +91,7 @@ None.

 **Outputs**

- ``resources/load_raw.csv``
+- ``resources/electricity_demand.csv``


 Rule ``retrieve_cost_data``
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@ -24,9 +24,9 @@ rule build_electricity_demand:
        countries=config["countries"],
        load=config["load"],
    input:
-        ancient(RESOURCES + "load_raw.csv"),
+        ancient("data/electricity_demand_raw.csv"),
    output:
-        RESOURCES + "load.csv",
+        RESOURCES + "electricity_demand.csv",
    log:
        LOGS + "build_electricity_demand.log",
    resources:
@ -417,7 +417,7 @@ rule add_electricity:
            if config["conventional"]["dynamic_fuel_price"]
            else []
        ),
-        load=RESOURCES + "load.csv",
+        load=RESOURCES + "electricity_demand.csv",
        nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
        ua_md_gdp="data/GDP_PPP_30arcsec_v3_mapped_default.csv",
    output:
--- a/rules/retrieve.smk
+++ b/rules/retrieve.smk
@ -188,27 +188,17 @@ if config["enable"]["retrieve"]:
 if config["enable"]["retrieve"]:

    rule retrieve_electricity_demand:
-        input:
-            HTTP.remote(
-                "data.open-power-system-data.org/time_series/{version}/time_series_60min_singleindex.csv".format(
-                    version=(
-                        "2019-06-05"
-                        if config["snapshots"]["end"] < "2019"
-                        else "2020-10-06"
-                    )
-                ),
-                keep_local=True,
-                static=True,
-            ),
+        params:
+            versions=["2019-06-05", "2020-10-06"],
        output:
-            RESOURCES + "load_raw.csv",
+            "data/electricity_demand_raw.csv",
        log:
            LOGS + "retrieve_electricity_demand.log",
        resources:
            mem_mb=5000,
        retries: 2
-        run:
-            move(input[0], output[0])
+        script:
+            "../scripts/retrieve_electricity_demand.py"


 if config["enable"]["retrieve"]:
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@ -52,7 +52,7 @@ Inputs
        :scale: 34 %

 - ``data/geth2015_hydro_capacities.csv``: alternative to capacities above; not currently used!
- ``resources/load.csv`` Hourly per-country load profiles.
+- ``resources/electricity_demand.csv`` Hourly per-country electricity demand profiles.
 - ``resources/regions_onshore.geojson``: confer :ref:`busregions`
 - ``resources/nuts3_shapes.geojson``: confer :ref:`shapes`
 - ``resources/powerplants.csv``: confer :ref:`powerplants`
--- a/scripts/build_electricity_demand.py
+++ b/scripts/build_electricity_demand.py
@ -1,15 +1,13 @@
 # -*- coding: utf-8 -*-
-# SPDX-FileCopyrightText: : 2020 @JanFrederickUnnewehr, The PyPSA-Eur Authors
+# SPDX-FileCopyrightText: : 2020 @JanFrederickUnnewehr, 2020-2024 The PyPSA-Eur Authors
 #
 # SPDX-License-Identifier: MIT
 """
-This rule downloads the load data from `Open Power System Data Time series.
-
+This rule downloads the load data from `Open Power System Data Time series
 <https://data.open-power-system-data.org/time_series/>`_. For all countries in
-the network, the per country load timeseries with suffix
-``_load_actual_entsoe_transparency`` are extracted from the dataset. After
-filling small gaps linearly and large gaps by copying time-slice of a given
-period, the load data is exported to a ``.csv`` file.
+the network, the per country load timeseries are extracted from the dataset.
+After filling small gaps linearly and large gaps by copying time-slice of a
+given period, the load data is exported to a ``.csv`` file.

 Relevant Settings
 -----------------
@ -19,9 +17,7 @@ Relevant Settings
    snapshots:

    load:
-        interpolate_limit:
-        time_shift_for_large_gaps:
-        manual_adjustments:
+        interpolate_limit: time_shift_for_large_gaps: manual_adjustments:


 .. seealso::
@ -31,12 +27,12 @@ Relevant Settings
 Inputs
 ------

- ``resources/load_raw.csv``:
+- ``data/electricity_demand_raw.csv``:

 Outputs
 -------

- ``resources/load.csv``:
+- ``resources/electricity_demand.csv``:
 """

 import logging
@ -49,7 +45,7 @@ from pandas import Timedelta as Delta
 logger = logging.getLogger(__name__)


-def load_timeseries(fn, years, countries, powerstatistics=True):
+def load_timeseries(fn, years, countries):
    """
    Read load data from OPSD time-series package version 2020-10-06.

@ -62,29 +58,15 @@ def load_timeseries(fn, years, countries, powerstatistics=True):
        File name or url location (file format .csv)
    countries : listlike
        Countries for which to read load data.
-    powerstatistics: bool
-        Whether the electricity consumption data of the ENTSOE power
-        statistics (if true) or of the ENTSOE transparency map (if false)
-        should be parsed.

    Returns
    -------
    load : pd.DataFrame
        Load time-series with UTC timestamps x ISO-2 countries
    """
-    logger.info(f"Retrieving load data from '{fn}'.")
-
-    pattern = "power_statistics" if powerstatistics else "transparency"
-    pattern = f"_load_actual_entsoe_{pattern}"
-
-    def rename(s):
-        return s[: -len(pattern)]
-
    return (
        pd.read_csv(fn, index_col=0, parse_dates=[0], date_format="%Y-%m-%dT%H:%M:%SZ")
        .tz_localize(None)
-        .filter(like=pattern)
-        .rename(columns=rename)
        .dropna(how="all", axis=0)
        .rename(columns={"GB_UKM": "GB"})
        .filter(items=countries)
@ -149,17 +131,18 @@ def copy_timeslice(load, cntry, start, stop, delta, fn_load=None):
            ].values
        elif fn_load is not None:
            duration = pd.date_range(freq="h", start=start - delta, end=stop - delta)
-            load_raw = load_timeseries(fn_load, duration, [cntry], powerstatistics)
+            load_raw = load_timeseries(fn_load, duration, [cntry])
            load.loc[start:stop, cntry] = load_raw.loc[
                start - delta : stop - delta, cntry
            ].values


-def manual_adjustment(load, fn_load, powerstatistics, countries):
+def manual_adjustment(load, fn_load, countries):
    """
    Adjust gaps manual for load data from OPSD time-series package.

-     1. For the ENTSOE power statistics load data (if powerstatistics is True)
+     1. For years later than 2015 for which the load data is mainly taken from the
+        ENTSOE power statistics

     Kosovo (KV) and Albania (AL) do not exist in the data set. Kosovo gets the
     same load curve as Serbia and Albania the same as Macdedonia, both scaled
@ -167,7 +150,8 @@ def manual_adjustment(load, fn_load, powerstatistics, countries):
     IEA Data browser [0] for the year 2013.


-     2. For the ENTSOE transparency load data (if powerstatistics is False)
+     2. For years earlier than 2015 for which the load data is mainly taken from the
+        ENTSOE transparency platforms

     Albania (AL) and Macedonia (MK) do not exist in the data set. Both get the
     same load curve as Montenegro,  scaled by the corresponding ratio of total energy
@ -183,9 +167,6 @@ def manual_adjustment(load, fn_load, powerstatistics, countries):
     ----------
     load : pd.DataFrame
         Load time-series with UTC timestamps x ISO-2 countries
-     powerstatistics: bool
-         Whether argument load comprises the electricity consumption data of
-         the ENTSOE power statistics or of the ENTSOE transparency map
    load_fn: str
         File name or url location (file format .csv)

@ -195,88 +176,72 @@ def manual_adjustment(load, fn_load, powerstatistics, countries):
         Manual adjusted and interpolated load time-series with UTC
         timestamps x ISO-2 countries
    """
-    if powerstatistics:
-        if "MK" in load.columns:
-            if "AL" not in load.columns or load.AL.isnull().values.all():
-                load["AL"] = load["MK"] * (4.1 / 7.4)
-        if "RS" in load.columns:
-            if "KV" not in load.columns or load.KV.isnull().values.all():
-                load["KV"] = load["RS"] * (4.8 / 27.0)

-        copy_timeslice(
-            load, "GR", "2015-08-11 21:00", "2015-08-15 20:00", Delta(weeks=1)
-        )
-        copy_timeslice(
-            load, "AT", "2018-12-31 22:00", "2019-01-01 22:00", Delta(days=2)
-        )
-        copy_timeslice(
-            load, "CH", "2010-01-19 07:00", "2010-01-19 22:00", Delta(days=1)
-        )
-        copy_timeslice(
-            load, "CH", "2010-03-28 00:00", "2010-03-28 21:00", Delta(days=1)
-        )
-        # is a WE, so take WE before
-        copy_timeslice(
-            load, "CH", "2010-10-08 13:00", "2010-10-10 21:00", Delta(weeks=1)
-        )
-        copy_timeslice(
-            load, "CH", "2010-11-04 04:00", "2010-11-04 22:00", Delta(days=1)
-        )
-        copy_timeslice(
-            load, "NO", "2010-12-09 11:00", "2010-12-09 18:00", Delta(days=1)
-        )
-        # whole january missing
-        copy_timeslice(
-            load,
-            "GB",
-            "2010-01-01 00:00",
-            "2010-01-31 23:00",
-            Delta(days=-365),
-            fn_load,
-        )
-        # 1.1. at midnight gets special treatment
-        copy_timeslice(
-            load,
-            "IE",
-            "2016-01-01 00:00",
-            "2016-01-01 01:00",
-            Delta(days=-366),
-            fn_load,
-        )
-        copy_timeslice(
-            load,
-            "PT",
-            "2016-01-01 00:00",
-            "2016-01-01 01:00",
-            Delta(days=-366),
-            fn_load,
-        )
-        copy_timeslice(
-            load,
-            "GB",
-            "2016-01-01 00:00",
-            "2016-01-01 01:00",
-            Delta(days=-366),
-            fn_load,
-        )
-
-    else:
+    if "AL" not in load and "AL" in countries:
        if "ME" in load:
-            if "AL" not in load and "AL" in countries:
-                load["AL"] = load.ME * (5.7 / 2.9)
-            if "MK" not in load and "MK" in countries:
+            load["AL"] = load.ME * (5.7 / 2.9)
+        elif "MK" in load:
+            load["AL"] = load["MK"] * (4.1 / 7.4)
+
+    if "MK" in countries:
+        if "MK" not in load or load.MK.isnull().sum() > len(load) / 2:
+            if "ME" in load:
                load["MK"] = load.ME * (6.7 / 2.9)
-            if "BA" not in load and "BA" in countries:
-                load["BA"] = load.HR * (11.0 / 16.2)
-        copy_timeslice(
-            load, "BG", "2018-10-27 21:00", "2018-10-28 22:00", Delta(weeks=1)
-        )
-        copy_timeslice(
-            load, "LU", "2019-01-02 11:00", "2019-01-05 05:00", Delta(weeks=-1)
-        )
-        copy_timeslice(
-            load, "LU", "2019-02-05 20:00", "2019-02-06 19:00", Delta(weeks=-1)
-        )
+
+    if "BA" not in load and "BA" in countries:
+        if "ME" in load:
+            load["BA"] = load.HR * (11.0 / 16.2)
+
+    if "KV" not in load or load.KV.isnull().values.all():
+        if "RS" in load:
+            load["KV"] = load["RS"] * (4.8 / 27.0)
+
+    copy_timeslice(load, "GR", "2015-08-11 21:00", "2015-08-15 20:00", Delta(weeks=1))
+    copy_timeslice(load, "AT", "2018-12-31 22:00", "2019-01-01 22:00", Delta(days=2))
+    copy_timeslice(load, "CH", "2010-01-19 07:00", "2010-01-19 22:00", Delta(days=1))
+    copy_timeslice(load, "CH", "2010-03-28 00:00", "2010-03-28 21:00", Delta(days=1))
+    # is a WE, so take WE before
+    copy_timeslice(load, "CH", "2010-10-08 13:00", "2010-10-10 21:00", Delta(weeks=1))
+    copy_timeslice(load, "CH", "2010-11-04 04:00", "2010-11-04 22:00", Delta(days=1))
+    copy_timeslice(load, "NO", "2010-12-09 11:00", "2010-12-09 18:00", Delta(days=1))
+    # whole january missing
+    copy_timeslice(
+        load,
+        "GB",
+        "2010-01-01 00:00",
+        "2010-01-31 23:00",
+        Delta(days=-365),
+        fn_load,
+    )
+    # 1.1. at midnight gets special treatment
+    copy_timeslice(
+        load,
+        "IE",
+        "2016-01-01 00:00",
+        "2016-01-01 01:00",
+        Delta(days=-366),
+        fn_load,
+    )
+    copy_timeslice(
+        load,
+        "PT",
+        "2016-01-01 00:00",
+        "2016-01-01 01:00",
+        Delta(days=-366),
+        fn_load,
+    )
+    copy_timeslice(
+        load,
+        "GB",
+        "2016-01-01 00:00",
+        "2016-01-01 01:00",
+        Delta(days=-366),
+        fn_load,
+    )
+
+    copy_timeslice(load, "BG", "2018-10-27 21:00", "2018-10-28 22:00", Delta(weeks=1))
+    copy_timeslice(load, "LU", "2019-01-02 11:00", "2019-01-05 05:00", Delta(weeks=-1))
+    copy_timeslice(load, "LU", "2019-02-05 20:00", "2019-02-06 19:00", Delta(weeks=-1))

    if "UA" in countries:
        copy_timeslice(
@ -297,14 +262,13 @@ if __name__ == "__main__":

    configure_logging(snakemake)

-    powerstatistics = snakemake.params.load["power_statistics"]
    interpolate_limit = snakemake.params.load["interpolate_limit"]
    countries = snakemake.params.countries
    snapshots = pd.date_range(freq="h", **snakemake.params.snapshots)
    years = slice(snapshots[0], snapshots[-1])
    time_shift = snakemake.params.load["time_shift_for_large_gaps"]

-    load = load_timeseries(snakemake.input[0], years, countries, powerstatistics)
+    load = load_timeseries(snakemake.input[0], years, countries)

    if "UA" in countries:
        # attach load of UA (best data only for entsoe transparency)
@ -321,7 +285,7 @@ if __name__ == "__main__":
            load["MD"] = 6.2e6 * (load_ua / load_ua.sum())

    if snakemake.params.load["manual_adjustments"]:
-        load = manual_adjustment(load, snakemake.input[0], powerstatistics, countries)
+        load = manual_adjustment(load, snakemake.input[0], countries)

    if load.empty:
        logger.warning("Build electricity demand time series is empty.")
--- a/scripts/retrieve_electricity_demand.py
+++ b/scripts/retrieve_electricity_demand.py
@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: 2023-2024 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+"""
+Retrieve electricity prices from OPSD.
+"""
+
+import logging
+
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+from _helpers import configure_logging
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake("retrieve_electricity_demand")
+        rootpath = ".."
+    else:
+        rootpath = "."
+    configure_logging(snakemake)
+
+    url = "https://data.open-power-system-data.org/time_series/{version}/time_series_60min_singleindex.csv"
+
+    df1, df2 = [
+        pd.read_csv(url.format(version=version), index_col=0)
+        for version in snakemake.params.versions
+    ]
+    combined = pd.concat([df1, df2[df2.index > df1.index[-1]]])
+
+    pattern = "_load_actual_entsoe_transparency"
+    transparency = combined.filter(like=pattern).rename(
+        columns=lambda x: x.replace(pattern, "")
+    )
+    pattern = "_load_actual_entsoe_power_statistics"
+    powerstatistics = combined.filter(like=pattern).rename(
+        columns=lambda x: x.replace(pattern, "")
+    )
+
+    res = transparency.fillna(powerstatistics)
+
+    res.to_csv(snakemake.output[0])