Merge branch 'master' into multiyear

.github/workflows/ci.yaml

@@ -19,7 +19,7 @@ on:
   - cron: "0 5 * * TUE"
 
 env:
-  DATA_CACHE_NUMBER: 2
+  DATA_CACHE_NUMBER: 1
 
 jobs:
   build:

config/config.default.yaml

@@ -319,9 +319,8 @@ pypsa_eur:
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#energy
 energy:
-  energy_totals_year: 2013
+  energy_totals_year: 2019
   base_emissions_year: 1990
-  eurostat_report_year: 2016
   emissions: CO2
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#biomass
@@ -360,7 +359,7 @@ solar_thermal:
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#existing-capacities
 existing_capacities:
-  grouping_years_power: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030]
+  grouping_years_power: [1960, 1965, 1970, 1975, 1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030]
   grouping_years_heat: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019] # these should not extend 2020
   threshold_capacity: 10
   default_heating_lifetime: 20

data/switzerland-new_format-all_years.csv

@@ -1,25 +1,25 @@
-country,item,2010,2011,2012,2013,2014,2015
+country,item,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022
-CH,total residential,268.2,223.4,243.4,261.3,214.2,229.1
+CH,total residential,268.2,223.4,243.4,261.3,214.2,229.1,241.2,236.5,223.7,226.5,219.1,241.2,211.3
-CH,total residential space,192.2,149.0,168.1,185.5,139.7,154.4
+CH,total residential space,192.2,149,168.1,185.5,139.7,154.4,167.3,161.5,147.2,150.4,140.2,166.2,131.9
-CH,total residential water,32.2,31.6,31.9,32.2,31.7,31.9
+CH,total residential water,32.2,31.6,31.9,32.2,31.7,31.9,31.8,31.8,31.8,31.7,33.3,32.5,32.5
-CH,total residential cooking,9.3,9.3,9.3,9.4,9.5,9.6
+CH,total residential cooking,9.3,9.3,9.3,9.4,9.5,9.6,9.9,10,10.1,10.2,10.5,10.3,10.3
-CH,electricity residential,67.9,63.7,65.7,67.6,63.0,64.4
+CH,electricity residential,67.9,63.7,65.7,67.6,63,64.4,69.7,69.2,67.7,68.1,68.7,70.8,66.8
-CH,electricity residential space,15.9,12.8,14.3,15.8,12.3,13.5
+CH,electricity residential space,15.9,12.8,14.3,15.8,12.3,13.5,15.8,15.6,14.7,15.3,14.8,17.8,14.8
-CH,electricity residential water,8.8,8.5,8.5,8.6,8.5,8.6
+CH,electricity residential water,8.8,8.5,8.5,8.6,8.5,8.6,8.9,9,9.2,9.3,9.7,9.5,9.5
-CH,electricity residential cooking,4.9,4.9,4.9,4.9,5.0,5.0
+CH,electricity residential cooking,4.9,4.9,4.9,4.9,5,5,5,5.1,5.1,5.1,5.4,5.2,5.3
-CH,total services,145.9,127.4,136.7,144.0,124.5,132.5
+CH,total services,145.9,127.4,136.7,144,124.5,132.5,150.5,147.7,141.5,143.1,129.7,144.2,122.5
-CH,total services space,80.0,62.2,70.8,77.4,58.3,64.3
+CH,total services space,80,62.2,70.8,77.4,58.3,64.3,77,74.4,68.2,69.8,64.3,75.7,58.7
-CH,total services water,10.1,10.0,10.1,10.1,10.0,10.0
+CH,total services water,10.1,10,10.1,10.1,10,10,11.4,11.3,11.2,11.1,9.7,10.4,12
-CH,total services cooking,2.5,2.4,2.3,2.3,2.4,2.3
+CH,total services cooking,2.5,2.4,2.3,2.3,2.4,2.3,3.1,3.1,3.2,3.3,2.1,2.6,3.2
-CH,electricity services,60.5,59.2,60.3,61.4,60.3,62.6
+CH,electricity services,60.5,59.2,60.3,61.4,60.3,62.6,65.9,65.7,65.5,65.6,58.8,61.6,61.6
-CH,electricity services space,4.0,3.2,3.8,4.2,3.3,3.6
+CH,electricity services space,4,3.2,3.8,4.2,3.3,3.6,2.7,2.5,2.3,2.3,2.2,2.5,2.5
-CH,electricity services water,0.7,0.7,0.7,0.7,0.7,0.7
+CH,electricity services water,0.7,0.7,0.7,0.7,0.7,0.7,1.2,1.1,1.1,1.1,0.9,1,1
-CH,electricity services cooking,2.5,2.4,2.3,2.3,2.4,2.3
+CH,electricity services cooking,2.5,2.4,2.3,2.3,2.4,2.3,3.1,3.1,3.1,3.2,3.3,2.1,3.2
-CH,total rail,11.5,11.1,11.2,11.4,11.1,11.4
+CH,total rail,11.5,11.1,11.2,11.4,11.1,11.4,11.6,11.4,11.2,11,10.2,10.6,10.8
-CH,total road,199.4,200.4,200.4,201.2,202.0,203.1
+CH,total road,199.4,200.4,200.4,201.2,202,203.1,203.9,203.7,202.6,200.5,182.6,188.3,193.3
-CH,electricity road,0.,0.,0.,0.,0.,0.
+CH,electricity road,0,0,0,0,0,0,0.1,0.2,0.3,0.4,0.5,0.8,1.3
-CH,electricity rail,11.5,11.1,11.2,11.4,11.1,11.4
+CH,electricity rail,11.5,11.1,11.2,11.4,11.1,11.4,11.5,11.3,11.1,11,10.1,10.6,10.7
-CH,total domestic aviation,3.3,3.2,3.4,3.4,3.5,3.5
+CH,total domestic aviation,3.3,3.2,3.4,3.4,3.5,3.5,3.6,3.1,3.1,2.9,2.5,2.8,3
-CH,total international aviation,58.0,62.0,63.5,64.2,64.5,66.8
+CH,total international aviation,58,62,63.5,64.2,64.5,66.8,70.6,72.8,77.2,78.2,28.2,31.2,56.8
-CH,total domestic navigation,1.6,1.6,1.6,1.6,1.6,1.6
+CH,total domestic navigation,1.6,1.6,1.6,1.6,1.6,1.6,1.4,1.4,1.4,1.4,1.4,1.4,1.4
-CH,total international navigation,0.,0.,0.,0.,0.,0.
+CH,total international navigation,0,0,0,0,0,0,0,0,0,0,0,0,0

doc/configtables/energy.csv

@@ -1,7 +1,4 @@
 ,Unit,Values,Description
 energy_totals_year ,--,"{1990,1995,2000,2005,2010,2011,…} ",The year for the sector energy use. The year must be avaliable in the Eurostat report
 base_emissions_year ,--,"YYYY; e.g. 1990","The base year for the sector emissions. See `European Environment Agency (EEA) <https://www.eea.europa.eu/data-and-maps/data/national-emissions-reported-to-the-unfccc-and-to-the-eu-greenhouse-gas-monitoring-mechanism-16>`_."
-
-eurostat_report_year ,--,"{2016,2017,2018}","The publication year of the Eurostat report. 2016 includes Bosnia and Herzegovina, 2017 does not"
-
 emissions ,--,"{CO2, All greenhouse gases - (CO2 equivalent)}","Specify which sectoral emissions are taken into account. Data derived from EEA. Currently only CO2 is implemented."

doc/release_notes.rst

@@ -9,6 +9,23 @@ Release Notes
 
 Upcoming Release
 ================
+
+* Corrected a bug leading to power plants operating after their DateOut
+  (https://github.com/PyPSA/pypsa-eur/pull/958). Added additional grouping years
+  before 1980.
+
+* The Eurostat data was updated to the 2023 version in :mod:`build_energy_totals`.
+
+* The latest `Swiss energy totals
+  <https://www.bfe.admin.ch/bfe/de/home/versorgung/statistik-und-geodaten/energiestatistiken/energieverbrauch-nach-verwendungszweck.html/>`_
+  have been updated to the 2023 version.
+
+* The JRC-IDEES data is only available until 2015. For energy totals years (``energy: energy_totals_year``) after
+  2015, the data scaled using the ratio of Eurostat data reported for the energy
+  totals year and 2015.
+
+* The default energy totals year (``energy: energy_totals_year``) was updated to 2019.
+
 * Upgrade default techno-economic assumptions to ``technology-data`` v0.8.1.
 
 * Linearly interpolate missing investment periods in year-dependent

rules/build_sector.smk

@@ -291,7 +291,7 @@ rule build_energy_totals:
         swiss="data/switzerland-new_format-all_years.csv",
         idees="data/bundle-sector/jrc-idees-2015",
         district_heat_share="data/district_heat_share.csv",
-        eurostat="data/eurostat-energy_balances-june_2021_edition",
+        eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition",
     output:
         energy_name=resources("energy_totals.csv"),
         co2_name=resources("co2_totals.csv"),
@@ -508,7 +508,7 @@ rule build_industrial_production_per_country:
     input:
         ammonia_production=resources("ammonia_production.csv"),
         jrc="data/bundle-sector/jrc-idees-2015",
-        eurostat="data/bundle-sector/eurostat-energy_balances-may_2018_edition",
+        eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition",
     output:
         industrial_production_per_country=resources(
             "industrial_production_per_country.csv"
@@ -878,7 +878,6 @@ rule prepare_sector_network:
         countries=config_provider("countries"),
         adjustments=config_provider("adjustments", "sector"),
         emissions_scope=config_provider("energy", "emissions"),
-        eurostat_report_year=config_provider("energy", "eurostat_report_year"),
         RDIR=RDIR,
     input:
         unpack(input_profile_offwind),
@@ -909,7 +908,7 @@ rule prepare_sector_network:
         ),
         network=resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"),
         energy_totals_name=resources("energy_totals.csv"),
-        eurostat=input_eurostat,
+        eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition",
         pop_weighted_energy_totals=resources(
             "pop_weighted_energy_totals_s{simpl}_{clusters}.csv"
         ),

rules/common.smk

@@ -130,12 +130,6 @@ def has_internet_access(url="www.zenodo.org") -> bool:
         conn.close()
 
 
-def input_eurostat(w):
-    # 2016 includes BA, 2017 does not
-    report_year = config_provider("energy", "eurostat_report_year")(w)
-    return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition"
-
-
 def solved_previous_horizon(w):
     planning_horizons = config_provider("scenario", "planning_horizons")(w)
     i = planning_horizons.index(int(w.planning_horizons))

rules/postprocess.smk

@@ -237,7 +237,6 @@ rule plot_summary:
         countries=config_provider("countries"),
         planning_horizons=config_provider("scenario", "planning_horizons"),
         emissions_scope=config_provider("energy", "emissions"),
-        eurostat_report_year=config_provider("energy", "eurostat_report_year"),
         plotting=config_provider("plotting"),
         foresight=config_provider("foresight"),
         co2_budget=config_provider("co2_budget"),
@@ -247,7 +246,7 @@ rule plot_summary:
         costs=RESULTS + "csvs/costs.csv",
         energy=RESULTS + "csvs/energy.csv",
         balances=RESULTS + "csvs/supply_energy.csv",
-        eurostat=input_eurostat,
+        eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition",
         co2="data/bundle-sector/eea/UNFCCC_v23.csv",
     output:
         costs=RESULTS + "graphs/costs.pdf",

rules/retrieve.smk

@@ -135,21 +135,10 @@ if config["enable"]["retrieve"] and config["enable"].get(
         "h2_salt_caverns_GWh_per_sqkm.geojson",
     ]
 
-    # TODO: check which versions of eurostat to keep
-    datafolders = [
-        protected(
-            directory("data/bundle-sector/eurostat-energy_balances-june_2016_edition")
-        ),
-        protected(
-            directory("data/bundle-sector/eurostat-energy_balances-may_2018_edition")
-        ),
-        protected(directory("data/bundle-sector/jrc-idees-2015")),
-    ]
-
     rule retrieve_sector_databundle:
         output:
             protected(expand("data/bundle-sector/{files}", files=datafiles)),
-            *datafolders,
+            protected(directory("data/bundle-sector/jrc-idees-2015")),
         log:
             "logs/retrieve_sector_databundle.log",
         retries: 2
@@ -158,6 +147,15 @@ if config["enable"]["retrieve"] and config["enable"].get(
         script:
             "../scripts/retrieve_sector_databundle.py"
 
+    rule retrieve_eurostat_data:
+        output:
+            directory("data/eurostat/eurostat-energy_balances-april_2023_edition"),
+        log:
+            "logs/retrieve_eurostat_data.log",
+        retries: 2
+        script:
+            "../scripts/retrieve_eurostat_data.py"
+
 
 if config["enable"]["retrieve"]:
     datafiles = [

scripts/add_existing_baseyear.py

@@ -171,10 +171,6 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas
     phased_out = df_agg[df_agg["DateOut"] < baseyear].index
     df_agg.drop(phased_out, inplace=True)
 
-    # calculate remaining lifetime before phase-out (+1 because assuming
-    # phase out date at the end of the year)
-    df_agg["lifetime"] = df_agg.DateOut - df_agg.DateIn + 1
-
     # assign clustered bus
     busmap_s = pd.read_csv(snakemake.input.busmap_s, index_col=0).squeeze()
     busmap = pd.read_csv(snakemake.input.busmap, index_col=0).squeeze()
@@ -195,6 +191,10 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas
         grouping_years, np.digitize(df_agg.DateIn, grouping_years, right=True)
     )
 
+    # calculate (adjusted) remaining lifetime before phase-out (+1 because assuming
+    # phase out date at the end of the year)
+    df_agg["lifetime"] = df_agg.DateOut - df_agg["grouping_year"] + 1
+
     df = df_agg.pivot_table(
         index=["grouping_year", "Fueltype"],
         columns="cluster_bus",

scripts/build_energy_totals.py

@@ -8,6 +8,7 @@ Build total energy demands per country using JRC IDEES, eurostat, and EEA data.
 
 import logging
 import multiprocessing as mp
+import os
 from functools import partial
 
 import country_converter as coco
@@ -36,8 +37,6 @@ def reverse(dictionary):
     return {v: k for k, v in dictionary.items()}
 
 
-non_EU = ["NO", "CH", "ME", "MK", "RS", "BA", "AL"]
-
 idees_rename = {"GR": "EL", "GB": "UK"}
 
 eu28 = cc.EU28as("ISO2").ISO2.tolist()
@@ -70,60 +69,57 @@ to_ipcc = {
 }
 
 
-def eurostat_per_country(country):
+def build_eurostat(input_eurostat, countries, year):
-
-    country_fn = idees_rename.get(country, country)
-    fn = (
-        snakemake.input.eurostat
-        + f"/{country_fn}-Energy-balance-sheets-June-2021-edition.xlsb"
-    )
-
-    df = pd.read_excel(
-        fn,
-        sheet_name=None,
-        skiprows=4,
-        index_col=list(range(3)),
-        na_values=["+", "-", "=", "Z", ":"],
-    )
-
-    df.pop("Cover")
-
-    return pd.concat(df)
-
-
-def build_eurostat(countries, year=None):
     """
     Return multi-index for all countries' energy data in TWh/a.
     """
-
+    df = {}
-    nprocesses = snakemake.threads
+    countries = {idees_rename.get(country, country) for country in countries} - {"CH"}
-    tqdm_kwargs = dict(
+    for country in countries:
-        ascii=False,
+        filename = (
-        unit=" country",
+            f"{input_eurostat}/{country}-Energy-balance-sheets-April-2023-edition.xlsb"
-        total=len(countries),
-        desc="Build from eurostat database",
         )
-    with mp.Pool(processes=nprocesses) as pool:
+        sheet = pd.read_excel(
-        dfs = list(tqdm(pool.imap(eurostat_per_country, countries), **tqdm_kwargs))
+            filename,
+            engine="pyxlsb",
+            sheet_name=str(year),
+            skiprows=4,
+            index_col=list(range(4)),
+        )
+        df[country] = sheet
+    df = pd.concat(df, axis=0)
 
-    index_names = ["country", "year", "lvl1", "lvl2", "lvl3"]
+    # drop columns with all NaNs
-    df = pd.concat(dfs, keys=countries, names=index_names)
+    unnamed_cols = df.columns[df.columns.astype(str).str.startswith("Unnamed")]
+    df.drop(unnamed_cols, axis=1, inplace=True)
+    df.drop(year, axis=1, inplace=True)
 
-    df.dropna(how="all", axis=0, inplace=True)
+    # make numeric values where possible
-    df.dropna(how="all", axis=1, inplace=True)
+    df.replace("Z", 0, inplace=True)
-    df = df[df.index.get_level_values("lvl1") != "ktoe"]
+    df = df.apply(pd.to_numeric, errors="coerce")
+    df = df.select_dtypes(include=[np.number])
 
-    i = df.index.to_frame(index=False)
+    # write 'International aviation' to the 2nd level of the multiindex
-    i.loc[i.lvl2 == "Primary production", ["lvl1", "lvl3"]] = "Main"
+    int_avia = df.index.get_level_values(2) == "International aviation"
-    i.loc[i.lvl2 == "Gross electricity production", "lvl1"] = "Gross production"
+    temp = df.loc[int_avia]
-    i.ffill(inplace=True)
+    temp.index = pd.MultiIndex.from_frame(
-    df.index = pd.MultiIndex.from_frame(i)
+        temp.index.to_frame().fillna("International aviation")
+    )
+    df = pd.concat([temp, df.loc[~int_avia]])
 
-    df.drop(list(range(1990, 2020)), axis=1, inplace=True)
+    # Renaming some indices
-    df.drop("Unnamed: 7", axis=1, inplace=True)
+    index_rename = {
-    df.fillna(0.0, inplace=True)
+        "Households": "Residential",
+        "Commercial & public services": "Services",
+        "Domestic navigation": "Domestic Navigation",
+        "International maritime bunkers": "Bunkers",
+    }
+    columns_rename = {"Total": "Total all products", "UK": "GB"}
+    df.rename(index=index_rename, columns=columns_rename, inplace=True)
+    df.sort_index(inplace=True)
+    df.index.names = [None] * len(df.index.names)
 
-    # convert ktoe/a to TWh/a
+    # convert to TWh/a from ktoe/a
     df *= 11.63 / 1e3
 
     df.index = df.index.set_levels(df.index.levels[1].astype(int), level=1)
@@ -659,7 +655,8 @@ def build_eea_co2(input_co2, year=1990, emissions_scope="CO2"):
     return emissions / 1e3
 
 
-def build_eurostat_co2(countries, eurostat=None, year=1990):
+def build_eurostat_co2(input_eurostat, countries, year=1990):
+    eurostat = build_eurostat(input_eurostat, countries, year)
 
     if eurostat is None:
         df = build_eurostat(countries, year)
@@ -688,57 +685,17 @@ def build_co2_totals(countries, eea_co2, eurostat_co2):
 
     for ct in pd.Index(countries).intersection(["BA", "RS", "AL", "ME", "MK"]):
         mappings = {
-            "electricity": (
+            "electricity": (ct, "+", "Electricity & heat generation", np.nan),
-                ct,
+            "residential non-elec": (ct, "+", "+", "Residential"),
-                "Transformation input",
+            "services non-elec": (ct, "+", "+", "Services"),
-                "Electricity & heat generation",
+            "road non-elec": (ct, "+", "+", "Road"),
-                "Main",
+            "rail non-elec": (ct, "+", "+", "Rail"),
-            ),
+            "domestic navigation": (ct, "+", "+", "Domestic Navigation"),
-            "residential non-elec": (
+            "international navigation": (ct, "-", "Bunkers"),
-                ct,
+            "domestic aviation": (ct, "+", "+", "Domestic aviation"),
-                "Final energy consumption",
+            "international aviation": (ct, "-", "International aviation"),
-                "Other sectors",
-                "Households",
-            ),
-            "services non-elec": (
-                ct,
-                "Final energy consumption",
-                "Other sectors",
-                "Commercial & public services",
-            ),
-            "road non-elec": (
-                ct,
-                "Final energy consumption",
-                "Transport sector",
-                "Road",
-            ),
-            "rail non-elec": (
-                ct,
-                "Final energy consumption",
-                "Transport sector",
-                "Rail",
-            ),
-            "domestic navigation": (
-                ct,
-                "Final energy consumption",
-                "Transport sector",
-                "Domestic navigation",
-            ),
-            "international navigation": (ct, "Main", "International maritime bunkers"),
-            "domestic aviation": (
-                ct,
-                "Final energy consumption",
-                "Transport sector",
-                "Domestic aviation",
-            ),
-            "international aviation": (ct, "Main", "International aviation"),
             # does not include industrial process emissions or fuel processing/refining
-            "industrial non-elec": (
+            "industrial non-elec": (ct, "+", "Industry sector"),
-                ct,
-                "Final energy consumption",
-                "Industry sector",
-                "Non-energy use in industry sector",
-            ),
             # does not include non-energy emissions
             "agriculture": (eurostat_co2.index.get_level_values(0) == ct)
             & eurostat_co2.index.isin(["Agriculture & forestry", "Fishing"], level=3),
@@ -786,6 +743,131 @@ def build_transport_data(countries, population, idees):
     return transport_data
 
 
+def rescale_idees_from_eurostat(
+    idees_countries, energy, eurostat, input_eurostat, countries
+):
+    """
+    Takes JRC IDEES data from 2015 and rescales it by the ratio of the eurostat
+    data and the 2015 eurostat data.
+
+    missing data: ['passenger car efficiency', 'passenger cars']
+    """
+    main_cols = ["Total all products", "Electricity"]
+    # read in the eurostat data for 2015
+    eurostat_2015 = build_eurostat(input_eurostat, countries, 2015)[main_cols]
+    eurostat_year = eurostat[main_cols]
+    # calculate the ratio of the two data sets
+    ratio = eurostat_year / eurostat_2015
+    ratio = ratio.droplevel([1, 4])
+    cols_rename = {"Total all products": "total", "Electricity": "ele"}
+    index_rename = {v: k for k, v in idees_rename.items()}
+    ratio.rename(columns=cols_rename, index=index_rename, inplace=True)
+
+    mappings = {
+        "Residential": {
+            "total": [
+                "total residential space",
+                "total residential water",
+                "total residential cooking",
+                "total residential",
+                "derived heat residential",
+                "thermal uses residential",
+            ],
+            "elec": [
+                "electricity residential space",
+                "electricity residential water",
+                "electricity residential cooking",
+                "electricity residential",
+            ],
+        },
+        "Services": {
+            "total": [
+                "total services space",
+                "total services water",
+                "total services cooking",
+                "total services",
+                "derived heat services",
+                "thermal uses services",
+            ],
+            "elec": [
+                "electricity services space",
+                "electricity services water",
+                "electricity services cooking",
+                "electricity services",
+            ],
+        },
+        "Agriculture & forestry": {
+            "total": [
+                "total agriculture heat",
+                "total agriculture machinery",
+                "total agriculture",
+            ],
+            "elec": [
+                "total agriculture electricity",
+            ],
+        },
+        "Road": {
+            "total": [
+                "total road",
+                "total passenger cars",
+                "total other road passenger",
+                "total light duty road freight",
+            ],
+            "elec": [
+                "electricity road",
+                "electricity passenger cars",
+                "electricity other road passenger",
+                "electricity light duty road freight",
+            ],
+        },
+        "Rail": {
+            "total": [
+                "total rail",
+                "total rail passenger",
+                "total rail freight",
+            ],
+            "elec": [
+                "electricity rail",
+                "electricity rail passenger",
+                "electricity rail freight",
+            ],
+        },
+    }
+
+    avia_inter = [
+        "total aviation passenger",
+        "total aviation freight",
+        "total international aviation passenger",
+        "total international aviation freight",
+        "total international aviation",
+    ]
+    avia_domestic = [
+        "total domestic aviation passenger",
+        "total domestic aviation freight",
+        "total domestic aviation",
+    ]
+    navigation = [
+        "total domestic navigation",
+    ]
+
+    for country in idees_countries:
+        for sector, mapping in mappings.items():
+            sector_ratio = ratio.loc[(country, slice(None), sector)]
+
+            energy.loc[country, mapping["total"]] *= sector_ratio["total"].iloc[0]
+            energy.loc[country, mapping["elec"]] *= sector_ratio["ele"].iloc[0]
+
+        avi_d = ratio.loc[(country, slice(None), "Domestic aviation"), "total"]
+        avi_i = ratio.loc[(country, "International aviation", slice(None)), "total"]
+        energy.loc[country, avia_inter] *= avi_i.iloc[0]
+        energy.loc[country, avia_domestic] *= avi_d.iloc[0]
+
+        nav = ratio.loc[(country, slice(None), "Domestic Navigation"), "total"]
+        energy.loc[country, navigation] *= nav.iloc[0]
+
+    return energy
+
+
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
@@ -805,22 +887,32 @@ if __name__ == "__main__":
     countries_without_ch = pd.Index(countries).difference(["CH"])
 
     data_year = params["energy_totals_year"]
-    report_year = snakemake.params.energy["eurostat_report_year"]
     input_eurostat = snakemake.input.eurostat
-    eurostat = build_eurostat(countries_without_ch)
+    eurostat = build_eurostat(input_eurostat, countries, data_year)
-    swiss = build_swiss()
+    swiss = build_swiss(data_year)
-    idees = build_idees(idees_countries, data_year)
+    # data from idees only exists from 2000-2015. read in latest data and rescale later
+    idees = build_idees(idees_countries, min(2015, data_year))
 
     energy = build_energy_totals(countries, eurostat, swiss, idees)
+
+    if data_year > 2015:
+        logger.info("Data year is after 2015. Rescaling IDEES data based on eurostat.")
+        energy = rescale_idees_from_eurostat(
+            idees_countries, energy, eurostat, input_eurostat, countries
+        )
+
     energy.to_csv(snakemake.output.energy_name)
 
-    district_heat_share = build_district_heat_share(countries, idees)
+    # use rescaled idees data to calculate district heat share
+    district_heat_share = build_district_heat_share(
+        countries, energy.loc[idees_countries]
+    )
     district_heat_share.to_csv(snakemake.output.district_heat_share)
 
     base_year_emissions = params["base_emissions_year"]
     emissions_scope = snakemake.params.energy["emissions"]
     eea_co2 = build_eea_co2(snakemake.input.co2, base_year_emissions, emissions_scope)
-    eurostat_co2 = build_eurostat_co2(countries, eurostat, base_year_emissions)
+    eurostat_co2 = build_eurostat_co2(input_eurostat, countries, base_year_emissions)
 
     co2 = build_co2_totals(countries, eea_co2, eurostat_co2)
     co2.to_csv(snakemake.output.co2_name)

scripts/build_industrial_production_per_country.py

@@ -97,33 +97,18 @@ fields = {
     "Other Industrial Sectors": "Physical output (index)",
 }
 
-eb_names = {
-    "NO": "Norway",
-    "AL": "Albania",
-    "BA": "Bosnia and Herzegovina",
-    "MK": "FYR of Macedonia",
-    "GE": "Georgia",
-    "IS": "Iceland",
-    "KO": "Kosovo",
-    "MD": "Moldova",
-    "ME": "Montenegro",
-    "RS": "Serbia",
-    "UA": "Ukraine",
-    "TR": "Turkey",
-}
-
 eb_sectors = {
-    "Iron & steel industry": "Iron and steel",
+    "Iron & steel": "Iron and steel",
-    "Chemical and Petrochemical industry": "Chemicals Industry",
+    "Chemical & petrochemical": "Chemicals Industry",
-    "Non-ferrous metal industry": "Non-metallic mineral products",
+    "Non-ferrous metals": "Non-metallic mineral products",
-    "Paper, Pulp and Print": "Pulp, paper and printing",
+    "Paper, pulp & printing": "Pulp, paper and printing",
-    "Food and Tabacco": "Food, beverages and tobacco",
+    "Food, beverages & tobacco": "Food, beverages and tobacco",
-    "Non-metallic Minerals (Glass, pottery & building mat. Industry)": "Non Ferrous Metals",
+    "Non-metallic minerals": "Non Ferrous Metals",
-    "Transport Equipment": "Transport Equipment",
+    "Transport equipment": "Transport Equipment",
     "Machinery": "Machinery Equipment",
-    "Textile and Leather": "Textiles and leather",
+    "Textile & leather": "Textiles and leather",
-    "Wood and Wood Products": "Wood and wood products",
+    "Wood & wood products": "Wood and wood products",
-    "Non-specified (Industry)": "Other Industrial Sectors",
+    "Not elsewhere specified (industry)": "Other Industrial Sectors",
 }
 
 # TODO: this should go in a csv in `data`
@@ -160,12 +145,15 @@ def get_energy_ratio(country, eurostat_dir, jrc_dir, year):
         e_country = e_switzerland * tj_to_ktoe
     else:
         # estimate physical output, energy consumption in the sector and country
-        fn = f"{eurostat_dir}/{eb_names[country]}.XLSX"
+        fn = f"{eurostat_dir}/{country}-Energy-balance-sheets-April-2023-edition.xlsb"
-        with mute_print():
         df = pd.read_excel(
-                fn, sheet_name="2016", index_col=2, header=0, skiprows=1
+            fn,
-            ).squeeze("columns")
+            sheet_name=str(min(2021, year)),
-        e_country = df.loc[eb_sectors.keys(), "Total all products"].rename(eb_sectors)
+            index_col=2,
+            header=0,
+            skiprows=4,
+        )
+        e_country = df.loc[eb_sectors.keys(), "Total"].rename(eb_sectors)
 
     fn = f"{jrc_dir}/JRC-IDEES-2015_Industry_EU28.xlsx"
 

scripts/plot_summary.py

@@ -462,7 +462,6 @@ def plot_carbon_budget_distribution(input_eurostat, options):
     plt.rcParams["ytick.labelsize"] = 20
 
     emissions_scope = snakemake.params.emissions_scope
-    report_year = snakemake.params.eurostat_report_year
     input_co2 = snakemake.input.co2
 
     # historic emissions
@@ -472,7 +471,6 @@ def plot_carbon_budget_distribution(input_eurostat, options):
         input_eurostat,
         options,
         emissions_scope,
-        report_year,
         input_co2,
         year=1990,
     )

scripts/prepare_sector_network.py

@@ -248,7 +248,7 @@ def get(item, investment_year=None):
 
 
 def co2_emissions_year(
-    countries, input_eurostat, options, emissions_scope, report_year, input_co2, year
+    countries, input_eurostat, options, emissions_scope, input_co2, year
 ):
     """
     Calculate CO2 emissions in one specific year (e.g. 1990 or 2018).
@@ -258,11 +258,9 @@ def co2_emissions_year(
     # TODO: read Eurostat data from year > 2014
     # this only affects the estimation of CO2 emissions for BA, RS, AL, ME, MK
     if year > 2014:
-        eurostat_co2 = build_eurostat_co2(
+        eurostat_co2 = build_eurostat_co2(input_eurostat, countries, 2014)
-            input_eurostat, countries, report_year, year=2014
-        )
     else:
-        eurostat_co2 = build_eurostat_co2(input_eurostat, countries, report_year, year)
+        eurostat_co2 = build_eurostat_co2(input_eurostat, countries, year)
 
     co2_totals = build_co2_totals(countries, eea_co2, eurostat_co2)
 
@@ -277,9 +275,7 @@ def co2_emissions_year(
 
 
 # TODO: move to own rule with sector-opts wildcard?
-def build_carbon_budget(
+def build_carbon_budget(o, input_eurostat, fn, emissions_scope, input_co2, options):
-    o, input_eurostat, fn, emissions_scope, report_year, input_co2, options
-):
     """
     Distribute carbon budget following beta or exponential transition path.
     """
@@ -300,7 +296,6 @@ def build_carbon_budget(
         input_eurostat,
         options,
         emissions_scope,
-        report_year,
         input_co2,
         year=1990,
     )
@@ -311,7 +306,6 @@ def build_carbon_budget(
         input_eurostat,
         options,
         emissions_scope,
-        report_year,
         input_co2,
         year=2018,
     )
@@ -3678,14 +3672,12 @@ if __name__ == "__main__":
         fn = "results/" + snakemake.params.RDIR + "/csvs/carbon_budget_distribution.csv"
         if not os.path.exists(fn):
             emissions_scope = snakemake.params.emissions_scope
-            report_year = snakemake.params.eurostat_report_year
             input_co2 = snakemake.input.co2
             build_carbon_budget(
                 co2_budget,
                 snakemake.input.eurostat,
                 fn,
                 emissions_scope,
-                report_year,
                 input_co2,
                 options,
             )

scripts/retrieve_eurostat_data.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2024- The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+"""
+Retrieve and extract eurostat energy balances data.
+"""
+
+
+import logging
+import zipfile
+from pathlib import Path
+
+from _helpers import configure_logging, progress_retrieve, set_scenario_config
+
+logger = logging.getLogger(__name__)
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake("retrieve_eurostat_data")
+        rootpath = ".."
+    else:
+        rootpath = "."
+    configure_logging(snakemake)
+    set_scenario_config(snakemake)
+
+    disable_progress = snakemake.config["run"].get("disable_progressbar", False)
+    url_eurostat = "https://ec.europa.eu/eurostat/documents/38154/4956218/Balances-December2022.zip/f7cf0d19-5c0f-60ad-4e48-098a5ddd6e48?t=1671184070589"
+    tarball_fn = Path(f"{rootpath}/data/eurostat/eurostat_2023.zip")
+    to_fn = Path(
+        f"{rootpath}/data/eurostat/eurostat-energy_balances-april_2023_edition/"
+    )
+
+    logger.info(f"Downloading Eurostat data from '{url_eurostat}'.")
+    progress_retrieve(url_eurostat, tarball_fn, disable=disable_progress)
+
+    logger.info("Extracting Eurostat data.")
+    with zipfile.ZipFile(tarball_fn, "r") as zip_ref:
+        zip_ref.extractall(to_fn)
+
+    logger.info(f"Eurostat data available in '{to_fn}'.")