new energy totals from eurostat 2023 report scaling JRC IDEES data from 2015

config/config.default.yaml

@@ -314,9 +314,9 @@ pypsa_eur:
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#energy
 energy:
-  energy_totals_year: 2013
+  energy_totals_year: 2019
   base_emissions_year: 1990
-  eurostat_report_year: 2016
+  eurostat_report_year: 2023
   emissions: CO2
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#biomass

data/switzerland-new_format-all_years.csv

@@ -1,25 +1,25 @@
-country,item,2010,2011,2012,2013,2014,2015
+country,item,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022
-CH,total residential,268.2,223.4,243.4,261.3,214.2,229.1
+CH,total residential,268.2,223.4,243.4,261.3,214.2,229.1,241.2,236.5,223.7,226.5,219.1,241.2,211.3
-CH,total residential space,192.2,149.0,168.1,185.5,139.7,154.4
+CH,total residential space,192.2,149,168.1,185.5,139.7,154.4,167.3,161.5,147.2,150.4,140.2,166.2,131.9
-CH,total residential water,32.2,31.6,31.9,32.2,31.7,31.9
+CH,total residential water,32.2,31.6,31.9,32.2,31.7,31.9,31.8,31.8,31.8,31.7,33.3,32.5,32.5
-CH,total residential cooking,9.3,9.3,9.3,9.4,9.5,9.6
+CH,total residential cooking,9.3,9.3,9.3,9.4,9.5,9.6,9.9,10,10.1,10.2,10.5,10.3,10.3
-CH,electricity residential,67.9,63.7,65.7,67.6,63.0,64.4
+CH,electricity residential,67.9,63.7,65.7,67.6,63,64.4,69.7,69.2,67.7,68.1,68.7,70.8,66.8
-CH,electricity residential space,15.9,12.8,14.3,15.8,12.3,13.5
+CH,electricity residential space,15.9,12.8,14.3,15.8,12.3,13.5,15.8,15.6,14.7,15.3,14.8,17.8,14.8
-CH,electricity residential water,8.8,8.5,8.5,8.6,8.5,8.6
+CH,electricity residential water,8.8,8.5,8.5,8.6,8.5,8.6,8.9,9,9.2,9.3,9.7,9.5,9.5
-CH,electricity residential cooking,4.9,4.9,4.9,4.9,5.0,5.0
+CH,electricity residential cooking,4.9,4.9,4.9,4.9,5,5,5,5.1,5.1,5.1,5.4,5.2,5.3
-CH,total services,145.9,127.4,136.7,144.0,124.5,132.5
+CH,total services,145.9,127.4,136.7,144,124.5,132.5,150.5,147.7,141.5,143.1,129.7,144.2,122.5
-CH,total services space,80.0,62.2,70.8,77.4,58.3,64.3
+CH,total services space,80,62.2,70.8,77.4,58.3,64.3,77,74.4,68.2,69.8,64.3,75.7,58.7
-CH,total services water,10.1,10.0,10.1,10.1,10.0,10.0
+CH,total services water,10.1,10,10.1,10.1,10,10,11.4,11.3,11.2,11.1,9.7,10.4,12
-CH,total services cooking,2.5,2.4,2.3,2.3,2.4,2.3
+CH,total services cooking,2.5,2.4,2.3,2.3,2.4,2.3,3.1,3.1,3.2,3.3,2.1,2.6,3.2
-CH,electricity services,60.5,59.2,60.3,61.4,60.3,62.6
+CH,electricity services,60.5,59.2,60.3,61.4,60.3,62.6,65.9,65.7,65.5,65.6,58.8,61.6,61.6
-CH,electricity services space,4.0,3.2,3.8,4.2,3.3,3.6
+CH,electricity services space,4,3.2,3.8,4.2,3.3,3.6,2.7,2.5,2.3,2.3,2.2,2.5,2.5
-CH,electricity services water,0.7,0.7,0.7,0.7,0.7,0.7
+CH,electricity services water,0.7,0.7,0.7,0.7,0.7,0.7,1.2,1.1,1.1,1.1,0.9,1,1
-CH,electricity services cooking,2.5,2.4,2.3,2.3,2.4,2.3
+CH,electricity services cooking,2.5,2.4,2.3,2.3,2.4,2.3,3.1,3.1,3.1,3.2,3.3,2.1,3.2
-CH,total rail,11.5,11.1,11.2,11.4,11.1,11.4
+CH,total rail,11.5,11.1,11.2,11.4,11.1,11.4,11.6,11.4,11.2,11,10.2,10.6,10.8
-CH,total road,199.4,200.4,200.4,201.2,202.0,203.1
+CH,total road,199.4,200.4,200.4,201.2,202,203.1,203.9,203.7,202.6,200.5,182.6,188.3,193.3
-CH,electricity road,0.,0.,0.,0.,0.,0.
+CH,electricity road,0,0,0,0,0,0,0.1,0.2,0.3,0.4,0.5,0.8,1.3
-CH,electricity rail,11.5,11.1,11.2,11.4,11.1,11.4
+CH,electricity rail,11.5,11.1,11.2,11.4,11.1,11.4,11.5,11.3,11.1,11,10.1,10.6,10.7
-CH,total domestic aviation,3.3,3.2,3.4,3.4,3.5,3.5
+CH,total domestic aviation,3.3,3.2,3.4,3.4,3.5,3.5,3.6,3.1,3.1,2.9,2.5,2.8,3
-CH,total international aviation,58.0,62.0,63.5,64.2,64.5,66.8
+CH,total international aviation,58,62,63.5,64.2,64.5,66.8,70.6,72.8,77.2,78.2,28.2,31.2,56.8
-CH,total domestic navigation,1.6,1.6,1.6,1.6,1.6,1.6
+CH,total domestic navigation,1.6,1.6,1.6,1.6,1.6,1.6,1.4,1.4,1.4,1.4,1.4,1.4,1.4
-CH,total international navigation,0.,0.,0.,0.,0.,0.
+CH,total international navigation,0,0,0,0,0,0,0,0,0,0,0,0,0

rules/common.smk

@@ -131,10 +131,11 @@ def has_internet_access(url="www.zenodo.org") -> bool:
 
 
 def input_eurostat(w):
-    # 2016 includes BA, 2017 does not
+    if config["energy"]["eurostat_report_year"] != 2023:
-    report_year = config_provider("energy", "eurostat_report_year")(w)
+        report_year = config["energy"]["eurostat_report_year"]
-    return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition"
+        return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition"
-
+    else:
+        return "data/bundle-sector/eurostat-energy_balances-april_2023_edition"
 
 def solved_previous_horizon(w):
     planning_horizons = config_provider("scenario", "planning_horizons")(w)

scripts/build_energy_totals.py

@@ -16,6 +16,7 @@ import numpy as np
 import pandas as pd
 from _helpers import configure_logging, mute_print, set_scenario_config
 from tqdm import tqdm
+import os
 
 cc = coco.CountryConverter()
 logger = logging.getLogger(__name__)
@@ -120,36 +121,93 @@ def build_eurostat(input_eurostat, countries, report_year, year):
     """
     Return multi-index for all countries' energy data in TWh/a.
     """
-    filenames = {
+    if report_year != 2023:
+        filenames = {
         2016: f"/{year}-Energy-Balances-June2016edition.xlsx",
         2017: f"/{year}-ENERGY-BALANCES-June2017edition.xlsx",
     }
 
-    with mute_print():
+        with mute_print():
-        dfs = pd.read_excel(
+            dfs = pd.read_excel(
-            input_eurostat + filenames[report_year],
+                input_eurostat + filenames[report_year],
-            sheet_name=None,
+                sheet_name=None,
-            skiprows=1,
+                skiprows=1,
-            index_col=list(range(4)),
+                index_col=list(range(4)),
-        )
+            )
 
-    # sorted_index necessary for slicing
+        # sorted_index necessary for slicing
-    lookup = eurostat_codes
+        lookup = eurostat_codes
-    labelled_dfs = {
+        labelled_dfs = {
-        lookup[df.columns[0]]: df
+            lookup[df.columns[0]]: df
-        for df in dfs.values()
+            for df in dfs.values()
-        if lookup[df.columns[0]] in countries
+            if lookup[df.columns[0]] in countries
-    }
+        }
-    df = pd.concat(labelled_dfs, sort=True).sort_index()
+        df = pd.concat(labelled_dfs, sort=True).sort_index()
+        # drop non-numeric and country columns
+        non_numeric_cols = df.columns[df.dtypes != float]
+        country_cols = df.columns.intersection(lookup.keys())
+        to_drop = non_numeric_cols.union(country_cols)
+        df.drop(to_drop, axis=1, inplace=True)
 
-    # drop non-numeric and country columns
+        # convert ktoe/a to TWh/a
-    non_numeric_cols = df.columns[df.dtypes != float]
+        df *= 11.63 / 1e3
-    country_cols = df.columns.intersection(lookup.keys())
+    
-    to_drop = non_numeric_cols.union(country_cols)
+    else:
-    df.drop(to_drop, axis=1, inplace=True)
+        # read in every country file in countries
+        eurostat = pd.DataFrame()
+        countries = [country if country != 'GB' else 'UK' for country in countries]
+        countries = [country if country != 'GR' else 'EL' for country in countries]
+        for country in countries:
+            filename = f"/{country}-Energy-balance-sheets-April-2023-edition.xlsb"
+            if os.path.exists(input_eurostat + filename):
+                df = pd.read_excel(
+                    input_eurostat + filename,
+                    engine='pyxlsb',
+                    sheet_name=str(year),
+                    skiprows=4,
+                    index_col=list(range(4)))
+                # replace entry 'Z' with 0
+                df.replace('Z', 0, inplace=True)
+                # write 'International aviation' to the 2nd level of the multiindex
+                index_number = (df.index.get_level_values(1) == 'International aviation').argmax()
+                new_index = ('-', 'International aviation', 'International aviation', 'ktoe')
+                modified_index = list(df.index)
+                modified_index[index_number] = new_index
+                df.index = pd.MultiIndex.from_tuples(modified_index, names=df.index.names)
+                # drop the annoying subhead line
+                df.drop(df[df[year] == year].index, inplace=True)
+                # replace 'Z' with 0
+                df = df.replace('Z', 0)
+                # add country to the multiindex
+                new_tuple = [(country, *idx) for idx in df.index]
+                new_mindex = pd.MultiIndex.from_tuples(new_tuple, names=['country', None, 'name', None, 'unit'])
+                df.index = new_mindex
+                # make numeric values where possible
+                df = df.apply(pd.to_numeric, errors='coerce')
+                # drop non-numeric columns
+                non_numeric_cols = df.columns[df.dtypes != float]
+                df.drop(non_numeric_cols, axis=1, inplace=True)
+                # concatenate the dataframes
+                eurostat = pd.concat([eurostat, df], axis=0)
+        
+        eurostat.drop(["Unnamed: 4", year, "Unnamed: 6"], axis=1, inplace=True)
+        # Renaming some indices
+        rename = {
+            'Households': 'Residential',
+            'Commercial & public services': 'Services',
+            'Domestic navigation': 'Domestic Navigation'
+        }
+        for name, rename in rename.items():
+            eurostat.index = eurostat.index.set_levels(
+            eurostat.index.levels[3].where(eurostat.index.levels[3] != name, rename),
+            level=3)
+        new_index = eurostat.index.set_levels(eurostat.index.levels[2].where(eurostat.index.levels[2] != 'International maritime bunkers', 'Bunkers'), level=2)
+        eurostat.index = new_index
 
-    # convert ktoe/a to TWh/a
+        eurostat.rename(columns={'Total': 'Total all products'}, inplace=True)
-    df *= 11.63 / 1e3
+        eurostat.index = eurostat.index.set_levels(eurostat.index.levels[0].where(eurostat.index.levels[0] != 'UK', 'GB'), level=0)
+        
+        df = eurostat * 11.63 / 1e3
 
     return df
 
@@ -669,31 +727,49 @@ def build_eurostat_co2(input_eurostat, countries, report_year, year=1990):
     return eurostat.multiply(specific_emissions).sum(axis=1)
 
 
-def build_co2_totals(countries, eea_co2, eurostat_co2):
+def build_co2_totals(countries, eea_co2, eurostat_co2, report_year):
     co2 = eea_co2.reindex(countries)
 
     for ct in pd.Index(countries).intersection(["BA", "RS", "AL", "ME", "MK"]):
-        mappings = {
+        if report_year != 2023:
-            "electricity": (
+            mappings = {
-                ct,
+                "electricity": (
-                "+",
+                    ct,
-                "Conventional Thermal Power Stations",
+                    "+",
-                "of which From Coal",
+                    "Conventional Thermal Power Stations",
-            ),
+                    "of which From Coal",
-            "residential non-elec": (ct, "+", "+", "Residential"),
+                ),
-            "services non-elec": (ct, "+", "+", "Services"),
+                "residential non-elec": (ct, "+", "+", "Residential"),
-            "road non-elec": (ct, "+", "+", "Road"),
+                "services non-elec": (ct, "+", "+", "Services"),
-            "rail non-elec": (ct, "+", "+", "Rail"),
+                "road non-elec": (ct, "+", "+", "Road"),
-            "domestic navigation": (ct, "+", "+", "Domestic Navigation"),
+                "rail non-elec": (ct, "+", "+", "Rail"),
-            "international navigation": (ct, "-", "Bunkers"),
+                "domestic navigation": (ct, "+", "+", "Domestic Navigation"),
-            "domestic aviation": (ct, "+", "+", "Domestic aviation"),
+                "international navigation": (ct, "-", "Bunkers"),
-            "international aviation": (ct, "+", "+", "International aviation"),
+                "domestic aviation": (ct, "+", "+", "Domestic aviation"),
-            # does not include industrial process emissions or fuel processing/refining
+                "international aviation": (ct, "+", "+", "International aviation"),
-            "industrial non-elec": (ct, "+", "Industry"),
+                # does not include industrial process emissions or fuel processing/refining
-            # does not include non-energy emissions
+                "industrial non-elec": (ct, "+", "Industry"),
-            "agriculture": (eurostat_co2.index.get_level_values(0) == ct)
+                # does not include non-energy emissions
-            & eurostat_co2.index.isin(["Agriculture / Forestry", "Fishing"], level=3),
+                "agriculture": (eurostat_co2.index.get_level_values(0) == ct)
-        }
+                & eurostat_co2.index.isin(["Agriculture / Forestry", "Fishing"], level=3),
+            }
+        else:
+            mappings = {
+                "electricity": (ct, "+", "Electricity & heat generation", np.nan),
+                "residential non-elec": (ct, "+", "+", "Residential"),
+                "services non-elec": (ct, "+", "+", "Services"),
+                "road non-elec": (ct, "+", "+", "Road"),
+                "rail non-elec": (ct, "+", "+", "Rail"),
+                "domestic navigation": (ct, "+", "+", "Domestic Navigation"),
+                "international navigation": (ct, "-", "Bunkers"),
+                "domestic aviation": (ct, "+", "+", "Domestic aviation"),
+                "international aviation": (ct, "-", "International aviation"),
+                # does not include industrial process emissions or fuel processing/refining
+                "industrial non-elec": (ct, "+", "Industry sector"),
+                # does not include non-energy emissions
+                "agriculture": (eurostat_co2.index.get_level_values(0) == ct)
+                & eurostat_co2.index.isin(["Agriculture & forestry", "Fishing"], level=3),
+            }
 
         for i, mi in mappings.items():
             co2.at[ct, i] = eurostat_co2.loc[mi].sum()
@@ -736,6 +812,133 @@ def build_transport_data(countries, population, idees):
 
     return transport_data
 
+def rescale(idees_countries, energy, eurostat):
+    '''
+    Takes JRC IDEES data from 2015 and rescales it by the ratio of the 
+    eurostat data and the 2015 eurostat data.
+    missing data: ['passenger car efficiency', 'passenger cars']
+    '''
+    # read in the eurostat data for 2015
+    eurostat_2015 = build_eurostat(input_eurostat, countries, 2023, 2015)[["Total all products", "Electricity"]]
+    # eurostat_2015 = eurostat_2015.rename(index={'GB': 'UK'}, level=0)
+    eurostat_year = eurostat[["Total all products", "Electricity"]]
+    # calculate the ratio of the two data sets
+    ratio = eurostat_year / eurostat_2015
+    ratio = ratio.droplevel([1,4])
+    ratio.rename(columns={"Total all products": "total", "Electricity": "ele"}, inplace=True)
+    ratio = ratio.rename(index={"GB": "UK"}, level=0)
+
+    residential_total = [
+        "total residential space",
+        "total residential water",
+        "total residential cooking",
+        "total residential",
+        "derived heat residential",
+        "thermal uses residential",
+    ]
+    residential_ele = [
+        "electricity residential space",
+        "electricity residential water",
+        "electricity residential cooking",
+        "electricity residential",
+    ]
+
+    service_total = [
+        "total services space",
+        "total services water",
+        "total services cooking",
+        "total services",
+        "derived heat services",
+        "thermal uses services",
+    ]
+    service_ele = [
+        "electricity services space",
+        "electricity services water",
+        "electricity services cooking",
+        "electricity services",
+    ]
+
+    agri_total = [
+        "total agriculture heat",
+        "total agriculture machinery",
+        "total agriculture",
+    ]
+    agri_ele = [
+        "total agriculture electricity",
+    ]
+
+    road_total = [
+        "total road",
+        "total passenger cars",
+        "total other road passenger",
+        "total light duty road freight",
+    ]
+    road_ele = [
+        "electricity road",
+        "electricity passenger cars",
+        "electricity other road passenger",
+        "electricity light duty road freight",
+    ]
+
+    rail_total = [
+        "total rail",
+        "total rail passenger",
+        "total rail freight",
+    ]
+    rail_ele = [
+        "electricity rail",
+        "electricity rail passenger",
+        "electricity rail freight",
+    ]
+
+    avia_inter = [
+        'total aviation passenger',
+        'total aviation freight',
+        'total international aviation passenger',
+        'total international aviation freight',
+        'total international aviation'
+    ]
+    avia_domestic = [
+        'total domestic aviation passenger',
+        'total domestic aviation freight',
+        'total domestic aviation',
+    ]
+    navigation = [
+        "total domestic navigation",
+    ]
+    
+    idees_countries = idees_countries.repalce({'GB': 'UK', 'GR': 'EL'})
+
+    for country in idees_countries:
+        res = ratio.loc[(country, slice(None), 'Residential')]
+        energy.loc[country, residential_total] *= res[['total']].iloc[0,0]
+        energy.loc[country, residential_ele] *= res[['ele']].iloc[0,0]
+
+        ser = ratio.loc[(country, slice(None), 'Services')]
+        energy.loc[country, service_total] *= ser[['total']].iloc[0,0]
+        energy.loc[country, service_ele] *= ser[['ele']].iloc[0,0]
+
+        agri = ratio.loc[(country, slice(None), 'Agriculture & forestry')]
+        energy.loc[country, agri_total] *= agri[['total']].iloc[0,0]
+        energy.loc[country, agri_ele] *= agri[['ele']].iloc[0,0]
+
+        road = ratio.loc[(country, slice(None), 'Road')]
+        energy.loc[country, road_total] *= road[['total']].iloc[0,0]
+        energy.loc[country, road_ele] *= road[['ele']].iloc[0,0]
+
+        rail = ratio.loc[(country, slice(None), 'Rail')]
+        energy.loc[country, rail_total] *= rail[['total']].iloc[0,0]
+        energy.loc[country, rail_ele] *= rail[['ele']].iloc[0,0]
+
+        avi_d = ratio.loc[(country, slice(None), 'Domestic aviation')]
+        avi_i = ratio.loc[(country, 'International aviation', slice(None))]
+        energy.loc[country, avia_inter] *= avi_i[['total']].iloc[0,0]
+        energy.loc[country, avia_domestic] *= avi_d[['total']].iloc[0,0]
+
+        nav = ratio.loc[(country, slice(None), 'Domestic Navigation')]
+        energy.loc[country, navigation] *= nav[['total']].iloc[0,0]
+
+    return energy
 
 if __name__ == "__main__":
     if "snakemake" not in globals():
@@ -759,12 +962,22 @@ if __name__ == "__main__":
     input_eurostat = snakemake.input.eurostat
     eurostat = build_eurostat(input_eurostat, countries, report_year, data_year)
     swiss = build_swiss(data_year)
-    idees = build_idees(idees_countries, data_year)
+    # data from idees only exists for 2015
+    if data_year > 2015:
+        # read in latest data and rescale later
+        idees = build_idees(idees_countries, 2015)
+    else:
+        idees = build_idees(idees_countries, data_year)
 
     energy = build_energy_totals(countries, eurostat, swiss, idees)
+    
+    if data_year > 2015:
+        energy = rescale(idees_countries, energy, eurostat)
+    
     energy.to_csv(snakemake.output.energy_name)
 
-    district_heat_share = build_district_heat_share(countries, idees)
+    # use rescaled idees data to calculate district heat share
+    district_heat_share = build_district_heat_share(countries, energy.loc[idees_countries])
     district_heat_share.to_csv(snakemake.output.district_heat_share)
 
     base_year_emissions = params["base_emissions_year"]
@@ -774,7 +987,7 @@ if __name__ == "__main__":
         input_eurostat, countries, report_year, base_year_emissions
     )
 
-    co2 = build_co2_totals(countries, eea_co2, eurostat_co2)
+    co2 = build_co2_totals(countries, eea_co2, eurostat_co2, report_year)
     co2.to_csv(snakemake.output.co2_name)
 
     transport = build_transport_data(countries, population, idees)