Merge branch 'yerbol-akhmetov-update_households_energy'

doc/release_notes.rst

@@ -316,7 +316,12 @@ Upcoming Release
 
 * Mark downloaded files as ``ancient`` rather than ``protected``.
 
-* Fix file name enconding in optional rule :mod:`build_biomass_transport_costs` depending on the operating system.
+* Fix file name encoding in optional rule :mod:`build_biomass_transport_costs` depending on the operating system.
+
+* Energy balances for residential demands (space, water, cooking) in the
+  JRC-IDEES data are updated with newer values from eurostat based on a
+  disaggregated household energy balance.
+  (https://github.com/PyPSA/pypsa-eur/pull/1025)
 
 PyPSA-Eur 0.10.0 (19th February 2024)
 =====================================

rules/build_sector.smk

@@ -293,6 +293,7 @@ rule build_energy_totals:
         idees="data/bundle/jrc-idees-2015",
         district_heat_share="data/district_heat_share.csv",
         eurostat="data/eurostat/Balances-April2023",
+        eurostat_households="data/eurostat/eurostat-household_energy_balances-february_2024.csv",
     output:
         energy_name=resources("energy_totals.csv"),
         co2_name=resources("co2_totals.csv"),

rules/retrieve.smk

@@ -54,6 +54,15 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle",
         script:
             "../scripts/retrieve_eurostat_data.py"
 
+    rule retrieve_eurostat_household_data:
+        output:
+            "data/eurostat/eurostat-household_energy_balances-february_2024.csv",
+        log:
+            "logs/retrieve_eurostat_household_data.log",
+        retries: 2
+        script:
+            "../scripts/retrieve_eurostat_household_data.py"
+
 
 if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True):
 

scripts/build_energy_totals.py

@@ -959,6 +959,48 @@ def rescale_idees_from_eurostat(
     return energy
 
 
+def update_residential_from_eurostat(energy):
+    """
+    Updates energy balances for residential from disaggregated data from
+    Eurostat.
+    """
+    eurostat_households = pd.read_csv(snakemake.input.eurostat_households)
+
+    # Column mapping for energy type
+    nrg_type = {
+        "total residential": ("FC_OTH_HH_E", "TOTAL"),
+        "total residential space": ("FC_OTH_HH_E_SH", "TOTAL"),
+        "total residential water": ("FC_OTH_HH_E_WH", "TOTAL"),
+        "total residential cooking": ("FC_OTH_HH_E_CK", "TOTAL"),
+        "electricity residential": ("FC_OTH_HH_E", "E7000"),
+        "electricity residential space": ("FC_OTH_HH_E_SH", "E7000"),
+        "electricity residential water": ("FC_OTH_HH_E_WH", "E7000"),
+        "electricity residential cooking": ("FC_OTH_HH_E_CK", "E7000"),
+    }
+
+    for nrg_name, (code, siec) in nrg_type.items():
+
+        # Select energy balance type, rename columns and countries to match IDEES data,
+        # convert TJ to TWh, and drop XK data already since included in RS data
+        col_to_rename = {"geo": "country", "TIME_PERIOD": "year", "OBS_VALUE": nrg_name}
+        idx_to_rename = {v: k for k, v in idees_rename.items()}
+        drop_geo = ["EU27_2020", "EA20", "XK"]
+        nrg_data = eurostat_households.query(
+            "nrg_bal == @code and siec == @siec and geo not in @drop_geo and OBS_VALUE > 0"
+        ).copy()
+        nrg_data.rename(columns=col_to_rename, inplace=True)
+        nrg_data = nrg_data.set_index(["country", "year"])[nrg_name] / 3.6e3
+        nrg_data.rename(index=idx_to_rename, inplace=True)
+
+        # update energy balance from household-specific eurostat data
+        idx = nrg_data.index.intersection(energy.index)
+        energy.loc[idx, nrg_name] = nrg_data[idx]
+
+    logger.info(
+        "Updated energy balances for residential using disaggregate final energy consumption data in Households from Eurostat"
+    )
+
+
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
@@ -992,6 +1034,8 @@ if __name__ == "__main__":
     logger.info("Extrapolate IDEES data based on eurostat for years 2015-2021.")
     energy = rescale_idees_from_eurostat(idees_countries, energy, eurostat)
 
+    update_residential_from_eurostat(energy)
+
     energy.to_csv(snakemake.output.energy_name)
 
     # use rescaled idees data to calculate district heat share

scripts/retrieve_eurostat_household_data.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2024- The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+"""
+Retrieve and extract eurostat household energy balances data.
+"""
+
+
+import gzip
+import logging
+import shutil
+from pathlib import Path
+
+from _helpers import configure_logging, progress_retrieve, set_scenario_config
+
+logger = logging.getLogger(__name__)
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake("retrieve_eurostat_data")
+        rootpath = ".."
+    else:
+        rootpath = "."
+    configure_logging(snakemake)
+    set_scenario_config(snakemake)
+
+    disable_progress = snakemake.config["run"].get("disable_progressbar", False)
+
+    url_eurostat_household = "https://ec.europa.eu/eurostat/databrowser-backend/api/extraction/1.0/LIVE/false/sdmx/csv/nrg_d_hhq__custom_11480365?startPeriod=2013&endPeriod=2022&i&compressed=true"
+    tarball_fn = Path(f"{rootpath}/data/eurostat/eurostat_household.gz")
+    to_fn = Path(
+        f"{rootpath}/data/eurostat/eurostat-household_energy_balances-february_2024.csv"
+    )
+
+    logger.info(
+        f"Downloading Eurostats' disaggregated household energy balances data from '{url_eurostat_household}'."
+    )
+    progress_retrieve(url_eurostat_household, tarball_fn, disable=disable_progress)
+
+    logger.info("Extracting Eurostat's disaggregated household energy balance data.")
+    with gzip.open(tarball_fn, "rb") as f_in, open(to_fn, "wb") as f_out:
+        shutil.copyfileobj(f_in, f_out)
+
+    logger.info(
+        f"Eurostat's disaggregated household energy balance data available in '{to_fn}'."
+    )