diff --git a/config/config.default.yaml b/config/config.default.yaml index ca2d1ba3..093b1aad 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -314,9 +314,9 @@ pypsa_eur: # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#energy energy: - energy_totals_year: 2013 + energy_totals_year: 2019 base_emissions_year: 1990 - eurostat_report_year: 2016 + eurostat_report_year: 2023 emissions: CO2 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#biomass diff --git a/data/switzerland-new_format-all_years.csv b/data/switzerland-new_format-all_years.csv index 93123009..d083e8a8 100644 --- a/data/switzerland-new_format-all_years.csv +++ b/data/switzerland-new_format-all_years.csv @@ -1,25 +1,25 @@ -country,item,2010,2011,2012,2013,2014,2015 -CH,total residential,268.2,223.4,243.4,261.3,214.2,229.1 -CH,total residential space,192.2,149.0,168.1,185.5,139.7,154.4 -CH,total residential water,32.2,31.6,31.9,32.2,31.7,31.9 -CH,total residential cooking,9.3,9.3,9.3,9.4,9.5,9.6 -CH,electricity residential,67.9,63.7,65.7,67.6,63.0,64.4 -CH,electricity residential space,15.9,12.8,14.3,15.8,12.3,13.5 -CH,electricity residential water,8.8,8.5,8.5,8.6,8.5,8.6 -CH,electricity residential cooking,4.9,4.9,4.9,4.9,5.0,5.0 -CH,total services,145.9,127.4,136.7,144.0,124.5,132.5 -CH,total services space,80.0,62.2,70.8,77.4,58.3,64.3 -CH,total services water,10.1,10.0,10.1,10.1,10.0,10.0 -CH,total services cooking,2.5,2.4,2.3,2.3,2.4,2.3 -CH,electricity services,60.5,59.2,60.3,61.4,60.3,62.6 -CH,electricity services space,4.0,3.2,3.8,4.2,3.3,3.6 -CH,electricity services water,0.7,0.7,0.7,0.7,0.7,0.7 -CH,electricity services cooking,2.5,2.4,2.3,2.3,2.4,2.3 -CH,total rail,11.5,11.1,11.2,11.4,11.1,11.4 -CH,total road,199.4,200.4,200.4,201.2,202.0,203.1 -CH,electricity road,0.,0.,0.,0.,0.,0. -CH,electricity rail,11.5,11.1,11.2,11.4,11.1,11.4 -CH,total domestic aviation,3.3,3.2,3.4,3.4,3.5,3.5 -CH,total international aviation,58.0,62.0,63.5,64.2,64.5,66.8 -CH,total domestic navigation,1.6,1.6,1.6,1.6,1.6,1.6 -CH,total international navigation,0.,0.,0.,0.,0.,0. +country,item,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022 +CH,total residential,268.2,223.4,243.4,261.3,214.2,229.1,241.2,236.5,223.7,226.5,219.1,241.2,211.3 +CH,total residential space,192.2,149,168.1,185.5,139.7,154.4,167.3,161.5,147.2,150.4,140.2,166.2,131.9 +CH,total residential water,32.2,31.6,31.9,32.2,31.7,31.9,31.8,31.8,31.8,31.7,33.3,32.5,32.5 +CH,total residential cooking,9.3,9.3,9.3,9.4,9.5,9.6,9.9,10,10.1,10.2,10.5,10.3,10.3 +CH,electricity residential,67.9,63.7,65.7,67.6,63,64.4,69.7,69.2,67.7,68.1,68.7,70.8,66.8 +CH,electricity residential space,15.9,12.8,14.3,15.8,12.3,13.5,15.8,15.6,14.7,15.3,14.8,17.8,14.8 +CH,electricity residential water,8.8,8.5,8.5,8.6,8.5,8.6,8.9,9,9.2,9.3,9.7,9.5,9.5 +CH,electricity residential cooking,4.9,4.9,4.9,4.9,5,5,5,5.1,5.1,5.1,5.4,5.2,5.3 +CH,total services,145.9,127.4,136.7,144,124.5,132.5,150.5,147.7,141.5,143.1,129.7,144.2,122.5 +CH,total services space,80,62.2,70.8,77.4,58.3,64.3,77,74.4,68.2,69.8,64.3,75.7,58.7 +CH,total services water,10.1,10,10.1,10.1,10,10,11.4,11.3,11.2,11.1,9.7,10.4,12 +CH,total services cooking,2.5,2.4,2.3,2.3,2.4,2.3,3.1,3.1,3.2,3.3,2.1,2.6,3.2 +CH,electricity services,60.5,59.2,60.3,61.4,60.3,62.6,65.9,65.7,65.5,65.6,58.8,61.6,61.6 +CH,electricity services space,4,3.2,3.8,4.2,3.3,3.6,2.7,2.5,2.3,2.3,2.2,2.5,2.5 +CH,electricity services water,0.7,0.7,0.7,0.7,0.7,0.7,1.2,1.1,1.1,1.1,0.9,1,1 +CH,electricity services cooking,2.5,2.4,2.3,2.3,2.4,2.3,3.1,3.1,3.1,3.2,3.3,2.1,3.2 +CH,total rail,11.5,11.1,11.2,11.4,11.1,11.4,11.6,11.4,11.2,11,10.2,10.6,10.8 +CH,total road,199.4,200.4,200.4,201.2,202,203.1,203.9,203.7,202.6,200.5,182.6,188.3,193.3 +CH,electricity road,0,0,0,0,0,0,0.1,0.2,0.3,0.4,0.5,0.8,1.3 +CH,electricity rail,11.5,11.1,11.2,11.4,11.1,11.4,11.5,11.3,11.1,11,10.1,10.6,10.7 +CH,total domestic aviation,3.3,3.2,3.4,3.4,3.5,3.5,3.6,3.1,3.1,2.9,2.5,2.8,3 +CH,total international aviation,58,62,63.5,64.2,64.5,66.8,70.6,72.8,77.2,78.2,28.2,31.2,56.8 +CH,total domestic navigation,1.6,1.6,1.6,1.6,1.6,1.6,1.4,1.4,1.4,1.4,1.4,1.4,1.4 +CH,total international navigation,0,0,0,0,0,0,0,0,0,0,0,0,0 diff --git a/rules/common.smk b/rules/common.smk index 618d746b..c3ce845c 100644 --- a/rules/common.smk +++ b/rules/common.smk @@ -131,10 +131,11 @@ def has_internet_access(url="www.zenodo.org") -> bool: def input_eurostat(w): - # 2016 includes BA, 2017 does not - report_year = config_provider("energy", "eurostat_report_year")(w) - return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition" - + if config["energy"]["eurostat_report_year"] != 2023: + report_year = config["energy"]["eurostat_report_year"] + return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition" + else: + return "data/bundle-sector/eurostat-energy_balances-april_2023_edition" def solved_previous_horizon(w): planning_horizons = config_provider("scenario", "planning_horizons")(w) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 960d1bbe..13a8c30d 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -16,6 +16,7 @@ import numpy as np import pandas as pd from _helpers import configure_logging, mute_print, set_scenario_config from tqdm import tqdm +import os cc = coco.CountryConverter() logger = logging.getLogger(__name__) @@ -120,36 +121,93 @@ def build_eurostat(input_eurostat, countries, report_year, year): """ Return multi-index for all countries' energy data in TWh/a. """ - filenames = { + if report_year != 2023: + filenames = { 2016: f"/{year}-Energy-Balances-June2016edition.xlsx", 2017: f"/{year}-ENERGY-BALANCES-June2017edition.xlsx", } - with mute_print(): - dfs = pd.read_excel( - input_eurostat + filenames[report_year], - sheet_name=None, - skiprows=1, - index_col=list(range(4)), - ) + with mute_print(): + dfs = pd.read_excel( + input_eurostat + filenames[report_year], + sheet_name=None, + skiprows=1, + index_col=list(range(4)), + ) - # sorted_index necessary for slicing - lookup = eurostat_codes - labelled_dfs = { - lookup[df.columns[0]]: df - for df in dfs.values() - if lookup[df.columns[0]] in countries - } - df = pd.concat(labelled_dfs, sort=True).sort_index() + # sorted_index necessary for slicing + lookup = eurostat_codes + labelled_dfs = { + lookup[df.columns[0]]: df + for df in dfs.values() + if lookup[df.columns[0]] in countries + } + df = pd.concat(labelled_dfs, sort=True).sort_index() + # drop non-numeric and country columns + non_numeric_cols = df.columns[df.dtypes != float] + country_cols = df.columns.intersection(lookup.keys()) + to_drop = non_numeric_cols.union(country_cols) + df.drop(to_drop, axis=1, inplace=True) - # drop non-numeric and country columns - non_numeric_cols = df.columns[df.dtypes != float] - country_cols = df.columns.intersection(lookup.keys()) - to_drop = non_numeric_cols.union(country_cols) - df.drop(to_drop, axis=1, inplace=True) + # convert ktoe/a to TWh/a + df *= 11.63 / 1e3 + + else: + # read in every country file in countries + eurostat = pd.DataFrame() + countries = [country if country != 'GB' else 'UK' for country in countries] + countries = [country if country != 'GR' else 'EL' for country in countries] + for country in countries: + filename = f"/{country}-Energy-balance-sheets-April-2023-edition.xlsb" + if os.path.exists(input_eurostat + filename): + df = pd.read_excel( + input_eurostat + filename, + engine='pyxlsb', + sheet_name=str(year), + skiprows=4, + index_col=list(range(4))) + # replace entry 'Z' with 0 + df.replace('Z', 0, inplace=True) + # write 'International aviation' to the 2nd level of the multiindex + index_number = (df.index.get_level_values(1) == 'International aviation').argmax() + new_index = ('-', 'International aviation', 'International aviation', 'ktoe') + modified_index = list(df.index) + modified_index[index_number] = new_index + df.index = pd.MultiIndex.from_tuples(modified_index, names=df.index.names) + # drop the annoying subhead line + df.drop(df[df[year] == year].index, inplace=True) + # replace 'Z' with 0 + df = df.replace('Z', 0) + # add country to the multiindex + new_tuple = [(country, *idx) for idx in df.index] + new_mindex = pd.MultiIndex.from_tuples(new_tuple, names=['country', None, 'name', None, 'unit']) + df.index = new_mindex + # make numeric values where possible + df = df.apply(pd.to_numeric, errors='coerce') + # drop non-numeric columns + non_numeric_cols = df.columns[df.dtypes != float] + df.drop(non_numeric_cols, axis=1, inplace=True) + # concatenate the dataframes + eurostat = pd.concat([eurostat, df], axis=0) + + eurostat.drop(["Unnamed: 4", year, "Unnamed: 6"], axis=1, inplace=True) + # Renaming some indices + rename = { + 'Households': 'Residential', + 'Commercial & public services': 'Services', + 'Domestic navigation': 'Domestic Navigation' + } + for name, rename in rename.items(): + eurostat.index = eurostat.index.set_levels( + eurostat.index.levels[3].where(eurostat.index.levels[3] != name, rename), + level=3) + new_index = eurostat.index.set_levels(eurostat.index.levels[2].where(eurostat.index.levels[2] != 'International maritime bunkers', 'Bunkers'), level=2) + eurostat.index = new_index - # convert ktoe/a to TWh/a - df *= 11.63 / 1e3 + eurostat.rename(columns={'Total': 'Total all products'}, inplace=True) + eurostat.index = eurostat.index.set_levels(eurostat.index.levels[0].where(eurostat.index.levels[0] != 'UK', 'GB'), level=0) + + df = eurostat * 11.63 / 1e3 return df @@ -669,31 +727,49 @@ def build_eurostat_co2(input_eurostat, countries, report_year, year=1990): return eurostat.multiply(specific_emissions).sum(axis=1) -def build_co2_totals(countries, eea_co2, eurostat_co2): +def build_co2_totals(countries, eea_co2, eurostat_co2, report_year): co2 = eea_co2.reindex(countries) for ct in pd.Index(countries).intersection(["BA", "RS", "AL", "ME", "MK"]): - mappings = { - "electricity": ( - ct, - "+", - "Conventional Thermal Power Stations", - "of which From Coal", - ), - "residential non-elec": (ct, "+", "+", "Residential"), - "services non-elec": (ct, "+", "+", "Services"), - "road non-elec": (ct, "+", "+", "Road"), - "rail non-elec": (ct, "+", "+", "Rail"), - "domestic navigation": (ct, "+", "+", "Domestic Navigation"), - "international navigation": (ct, "-", "Bunkers"), - "domestic aviation": (ct, "+", "+", "Domestic aviation"), - "international aviation": (ct, "+", "+", "International aviation"), - # does not include industrial process emissions or fuel processing/refining - "industrial non-elec": (ct, "+", "Industry"), - # does not include non-energy emissions - "agriculture": (eurostat_co2.index.get_level_values(0) == ct) - & eurostat_co2.index.isin(["Agriculture / Forestry", "Fishing"], level=3), - } + if report_year != 2023: + mappings = { + "electricity": ( + ct, + "+", + "Conventional Thermal Power Stations", + "of which From Coal", + ), + "residential non-elec": (ct, "+", "+", "Residential"), + "services non-elec": (ct, "+", "+", "Services"), + "road non-elec": (ct, "+", "+", "Road"), + "rail non-elec": (ct, "+", "+", "Rail"), + "domestic navigation": (ct, "+", "+", "Domestic Navigation"), + "international navigation": (ct, "-", "Bunkers"), + "domestic aviation": (ct, "+", "+", "Domestic aviation"), + "international aviation": (ct, "+", "+", "International aviation"), + # does not include industrial process emissions or fuel processing/refining + "industrial non-elec": (ct, "+", "Industry"), + # does not include non-energy emissions + "agriculture": (eurostat_co2.index.get_level_values(0) == ct) + & eurostat_co2.index.isin(["Agriculture / Forestry", "Fishing"], level=3), + } + else: + mappings = { + "electricity": (ct, "+", "Electricity & heat generation", np.nan), + "residential non-elec": (ct, "+", "+", "Residential"), + "services non-elec": (ct, "+", "+", "Services"), + "road non-elec": (ct, "+", "+", "Road"), + "rail non-elec": (ct, "+", "+", "Rail"), + "domestic navigation": (ct, "+", "+", "Domestic Navigation"), + "international navigation": (ct, "-", "Bunkers"), + "domestic aviation": (ct, "+", "+", "Domestic aviation"), + "international aviation": (ct, "-", "International aviation"), + # does not include industrial process emissions or fuel processing/refining + "industrial non-elec": (ct, "+", "Industry sector"), + # does not include non-energy emissions + "agriculture": (eurostat_co2.index.get_level_values(0) == ct) + & eurostat_co2.index.isin(["Agriculture & forestry", "Fishing"], level=3), + } for i, mi in mappings.items(): co2.at[ct, i] = eurostat_co2.loc[mi].sum() @@ -736,6 +812,133 @@ def build_transport_data(countries, population, idees): return transport_data +def rescale(idees_countries, energy, eurostat): + ''' + Takes JRC IDEES data from 2015 and rescales it by the ratio of the + eurostat data and the 2015 eurostat data. + missing data: ['passenger car efficiency', 'passenger cars'] + ''' + # read in the eurostat data for 2015 + eurostat_2015 = build_eurostat(input_eurostat, countries, 2023, 2015)[["Total all products", "Electricity"]] + # eurostat_2015 = eurostat_2015.rename(index={'GB': 'UK'}, level=0) + eurostat_year = eurostat[["Total all products", "Electricity"]] + # calculate the ratio of the two data sets + ratio = eurostat_year / eurostat_2015 + ratio = ratio.droplevel([1,4]) + ratio.rename(columns={"Total all products": "total", "Electricity": "ele"}, inplace=True) + ratio = ratio.rename(index={"GB": "UK"}, level=0) + + residential_total = [ + "total residential space", + "total residential water", + "total residential cooking", + "total residential", + "derived heat residential", + "thermal uses residential", + ] + residential_ele = [ + "electricity residential space", + "electricity residential water", + "electricity residential cooking", + "electricity residential", + ] + + service_total = [ + "total services space", + "total services water", + "total services cooking", + "total services", + "derived heat services", + "thermal uses services", + ] + service_ele = [ + "electricity services space", + "electricity services water", + "electricity services cooking", + "electricity services", + ] + + agri_total = [ + "total agriculture heat", + "total agriculture machinery", + "total agriculture", + ] + agri_ele = [ + "total agriculture electricity", + ] + + road_total = [ + "total road", + "total passenger cars", + "total other road passenger", + "total light duty road freight", + ] + road_ele = [ + "electricity road", + "electricity passenger cars", + "electricity other road passenger", + "electricity light duty road freight", + ] + + rail_total = [ + "total rail", + "total rail passenger", + "total rail freight", + ] + rail_ele = [ + "electricity rail", + "electricity rail passenger", + "electricity rail freight", + ] + + avia_inter = [ + 'total aviation passenger', + 'total aviation freight', + 'total international aviation passenger', + 'total international aviation freight', + 'total international aviation' + ] + avia_domestic = [ + 'total domestic aviation passenger', + 'total domestic aviation freight', + 'total domestic aviation', + ] + navigation = [ + "total domestic navigation", + ] + + idees_countries = idees_countries.repalce({'GB': 'UK', 'GR': 'EL'}) + + for country in idees_countries: + res = ratio.loc[(country, slice(None), 'Residential')] + energy.loc[country, residential_total] *= res[['total']].iloc[0,0] + energy.loc[country, residential_ele] *= res[['ele']].iloc[0,0] + + ser = ratio.loc[(country, slice(None), 'Services')] + energy.loc[country, service_total] *= ser[['total']].iloc[0,0] + energy.loc[country, service_ele] *= ser[['ele']].iloc[0,0] + + agri = ratio.loc[(country, slice(None), 'Agriculture & forestry')] + energy.loc[country, agri_total] *= agri[['total']].iloc[0,0] + energy.loc[country, agri_ele] *= agri[['ele']].iloc[0,0] + + road = ratio.loc[(country, slice(None), 'Road')] + energy.loc[country, road_total] *= road[['total']].iloc[0,0] + energy.loc[country, road_ele] *= road[['ele']].iloc[0,0] + + rail = ratio.loc[(country, slice(None), 'Rail')] + energy.loc[country, rail_total] *= rail[['total']].iloc[0,0] + energy.loc[country, rail_ele] *= rail[['ele']].iloc[0,0] + + avi_d = ratio.loc[(country, slice(None), 'Domestic aviation')] + avi_i = ratio.loc[(country, 'International aviation', slice(None))] + energy.loc[country, avia_inter] *= avi_i[['total']].iloc[0,0] + energy.loc[country, avia_domestic] *= avi_d[['total']].iloc[0,0] + + nav = ratio.loc[(country, slice(None), 'Domestic Navigation')] + energy.loc[country, navigation] *= nav[['total']].iloc[0,0] + + return energy if __name__ == "__main__": if "snakemake" not in globals(): @@ -759,12 +962,22 @@ if __name__ == "__main__": input_eurostat = snakemake.input.eurostat eurostat = build_eurostat(input_eurostat, countries, report_year, data_year) swiss = build_swiss(data_year) - idees = build_idees(idees_countries, data_year) + # data from idees only exists for 2015 + if data_year > 2015: + # read in latest data and rescale later + idees = build_idees(idees_countries, 2015) + else: + idees = build_idees(idees_countries, data_year) energy = build_energy_totals(countries, eurostat, swiss, idees) + + if data_year > 2015: + energy = rescale(idees_countries, energy, eurostat) + energy.to_csv(snakemake.output.energy_name) - district_heat_share = build_district_heat_share(countries, idees) + # use rescaled idees data to calculate district heat share + district_heat_share = build_district_heat_share(countries, energy.loc[idees_countries]) district_heat_share.to_csv(snakemake.output.district_heat_share) base_year_emissions = params["base_emissions_year"] @@ -774,7 +987,7 @@ if __name__ == "__main__": input_eurostat, countries, report_year, base_year_emissions ) - co2 = build_co2_totals(countries, eea_co2, eurostat_co2) + co2 = build_co2_totals(countries, eea_co2, eurostat_co2, report_year) co2.to_csv(snakemake.output.co2_name) transport = build_transport_data(countries, population, idees)