diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bad6039f..c17c0425 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,7 +19,7 @@ on: - cron: "0 5 * * TUE" env: - DATA_CACHE_NUMBER: 2 + DATA_CACHE_NUMBER: 1 jobs: build: diff --git a/config/config.default.yaml b/config/config.default.yaml index aa662839..23b13ddf 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -314,9 +314,8 @@ pypsa_eur: # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#energy energy: - energy_totals_year: 2013 + energy_totals_year: 2019 base_emissions_year: 1990 - eurostat_report_year: 2016 emissions: CO2 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#biomass @@ -354,7 +353,7 @@ solar_thermal: # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#existing-capacities existing_capacities: - grouping_years_power: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030] + grouping_years_power: [1960, 1965, 1970, 1975, 1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030] grouping_years_heat: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019] # these should not extend 2020 threshold_capacity: 10 default_heating_lifetime: 20 diff --git a/data/switzerland-new_format-all_years.csv b/data/switzerland-new_format-all_years.csv index 93123009..d083e8a8 100644 --- a/data/switzerland-new_format-all_years.csv +++ b/data/switzerland-new_format-all_years.csv @@ -1,25 +1,25 @@ -country,item,2010,2011,2012,2013,2014,2015 -CH,total residential,268.2,223.4,243.4,261.3,214.2,229.1 -CH,total residential space,192.2,149.0,168.1,185.5,139.7,154.4 -CH,total residential water,32.2,31.6,31.9,32.2,31.7,31.9 -CH,total residential cooking,9.3,9.3,9.3,9.4,9.5,9.6 -CH,electricity residential,67.9,63.7,65.7,67.6,63.0,64.4 -CH,electricity residential space,15.9,12.8,14.3,15.8,12.3,13.5 -CH,electricity residential water,8.8,8.5,8.5,8.6,8.5,8.6 -CH,electricity residential cooking,4.9,4.9,4.9,4.9,5.0,5.0 -CH,total services,145.9,127.4,136.7,144.0,124.5,132.5 -CH,total services space,80.0,62.2,70.8,77.4,58.3,64.3 -CH,total services water,10.1,10.0,10.1,10.1,10.0,10.0 -CH,total services cooking,2.5,2.4,2.3,2.3,2.4,2.3 -CH,electricity services,60.5,59.2,60.3,61.4,60.3,62.6 -CH,electricity services space,4.0,3.2,3.8,4.2,3.3,3.6 -CH,electricity services water,0.7,0.7,0.7,0.7,0.7,0.7 -CH,electricity services cooking,2.5,2.4,2.3,2.3,2.4,2.3 -CH,total rail,11.5,11.1,11.2,11.4,11.1,11.4 -CH,total road,199.4,200.4,200.4,201.2,202.0,203.1 -CH,electricity road,0.,0.,0.,0.,0.,0. -CH,electricity rail,11.5,11.1,11.2,11.4,11.1,11.4 -CH,total domestic aviation,3.3,3.2,3.4,3.4,3.5,3.5 -CH,total international aviation,58.0,62.0,63.5,64.2,64.5,66.8 -CH,total domestic navigation,1.6,1.6,1.6,1.6,1.6,1.6 -CH,total international navigation,0.,0.,0.,0.,0.,0. +country,item,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022 +CH,total residential,268.2,223.4,243.4,261.3,214.2,229.1,241.2,236.5,223.7,226.5,219.1,241.2,211.3 +CH,total residential space,192.2,149,168.1,185.5,139.7,154.4,167.3,161.5,147.2,150.4,140.2,166.2,131.9 +CH,total residential water,32.2,31.6,31.9,32.2,31.7,31.9,31.8,31.8,31.8,31.7,33.3,32.5,32.5 +CH,total residential cooking,9.3,9.3,9.3,9.4,9.5,9.6,9.9,10,10.1,10.2,10.5,10.3,10.3 +CH,electricity residential,67.9,63.7,65.7,67.6,63,64.4,69.7,69.2,67.7,68.1,68.7,70.8,66.8 +CH,electricity residential space,15.9,12.8,14.3,15.8,12.3,13.5,15.8,15.6,14.7,15.3,14.8,17.8,14.8 +CH,electricity residential water,8.8,8.5,8.5,8.6,8.5,8.6,8.9,9,9.2,9.3,9.7,9.5,9.5 +CH,electricity residential cooking,4.9,4.9,4.9,4.9,5,5,5,5.1,5.1,5.1,5.4,5.2,5.3 +CH,total services,145.9,127.4,136.7,144,124.5,132.5,150.5,147.7,141.5,143.1,129.7,144.2,122.5 +CH,total services space,80,62.2,70.8,77.4,58.3,64.3,77,74.4,68.2,69.8,64.3,75.7,58.7 +CH,total services water,10.1,10,10.1,10.1,10,10,11.4,11.3,11.2,11.1,9.7,10.4,12 +CH,total services cooking,2.5,2.4,2.3,2.3,2.4,2.3,3.1,3.1,3.2,3.3,2.1,2.6,3.2 +CH,electricity services,60.5,59.2,60.3,61.4,60.3,62.6,65.9,65.7,65.5,65.6,58.8,61.6,61.6 +CH,electricity services space,4,3.2,3.8,4.2,3.3,3.6,2.7,2.5,2.3,2.3,2.2,2.5,2.5 +CH,electricity services water,0.7,0.7,0.7,0.7,0.7,0.7,1.2,1.1,1.1,1.1,0.9,1,1 +CH,electricity services cooking,2.5,2.4,2.3,2.3,2.4,2.3,3.1,3.1,3.1,3.2,3.3,2.1,3.2 +CH,total rail,11.5,11.1,11.2,11.4,11.1,11.4,11.6,11.4,11.2,11,10.2,10.6,10.8 +CH,total road,199.4,200.4,200.4,201.2,202,203.1,203.9,203.7,202.6,200.5,182.6,188.3,193.3 +CH,electricity road,0,0,0,0,0,0,0.1,0.2,0.3,0.4,0.5,0.8,1.3 +CH,electricity rail,11.5,11.1,11.2,11.4,11.1,11.4,11.5,11.3,11.1,11,10.1,10.6,10.7 +CH,total domestic aviation,3.3,3.2,3.4,3.4,3.5,3.5,3.6,3.1,3.1,2.9,2.5,2.8,3 +CH,total international aviation,58,62,63.5,64.2,64.5,66.8,70.6,72.8,77.2,78.2,28.2,31.2,56.8 +CH,total domestic navigation,1.6,1.6,1.6,1.6,1.6,1.6,1.4,1.4,1.4,1.4,1.4,1.4,1.4 +CH,total international navigation,0,0,0,0,0,0,0,0,0,0,0,0,0 diff --git a/doc/configtables/energy.csv b/doc/configtables/energy.csv index 8718d75e..3d13b9c3 100644 --- a/doc/configtables/energy.csv +++ b/doc/configtables/energy.csv @@ -1,7 +1,4 @@ ,Unit,Values,Description energy_totals_year ,--,"{1990,1995,2000,2005,2010,2011,…} ",The year for the sector energy use. The year must be avaliable in the Eurostat report base_emissions_year ,--,"YYYY; e.g. 1990","The base year for the sector emissions. See `European Environment Agency (EEA) `_." - -eurostat_report_year ,--,"{2016,2017,2018}","The publication year of the Eurostat report. 2016 includes Bosnia and Herzegovina, 2017 does not" - emissions ,--,"{CO2, All greenhouse gases - (CO2 equivalent)}","Specify which sectoral emissions are taken into account. Data derived from EEA. Currently only CO2 is implemented." diff --git a/doc/release_notes.rst b/doc/release_notes.rst index c3da4147..44946133 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -9,11 +9,29 @@ Release Notes Upcoming Release ================ + * fix bug in land transport for temperature correction of ICEs and fuel cell cars * restructure land transport, demand is now attached to one single node, the different car types (ICE, EV, fuel cell) are modelled as links + +* Corrected a bug leading to power plants operating after their DateOut + (https://github.com/PyPSA/pypsa-eur/pull/958). Added additional grouping years + before 1980. + +* The Eurostat data was updated to the 2023 version in :mod:`build_energy_totals`. + +* The latest `Swiss energy totals + `_ + have been updated to the 2023 version. + +* The JRC-IDEES data is only available until 2015. For energy totals years (``energy: energy_totals_year``) after + 2015, the data scaled using the ratio of Eurostat data reported for the energy + totals year and 2015. + +* The default energy totals year (``energy: energy_totals_year``) was updated to 2019. + * Upgrade default techno-economic assumptions to ``technology-data`` v0.8.1. * Linearly interpolate missing investment periods in year-dependent diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 466d1713..9147a623 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -270,7 +270,7 @@ rule build_energy_totals: swiss="data/switzerland-new_format-all_years.csv", idees="data/bundle-sector/jrc-idees-2015", district_heat_share="data/district_heat_share.csv", - eurostat=input_eurostat, + eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition", output: energy_name=resources("energy_totals.csv"), co2_name=resources("co2_totals.csv"), @@ -468,7 +468,7 @@ rule build_industrial_production_per_country: input: ammonia_production=resources("ammonia_production.csv"), jrc="data/bundle-sector/jrc-idees-2015", - eurostat="data/bundle-sector/eurostat-energy_balances-may_2018_edition", + eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition", output: industrial_production_per_country=resources( "industrial_production_per_country.csv" @@ -834,7 +834,6 @@ rule prepare_sector_network: countries=config_provider("countries"), adjustments=config_provider("adjustments", "sector"), emissions_scope=config_provider("energy", "emissions"), - eurostat_report_year=config_provider("energy", "eurostat_report_year"), RDIR=RDIR, input: unpack(input_profile_offwind), @@ -865,7 +864,7 @@ rule prepare_sector_network: ), network=resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"), energy_totals_name=resources("energy_totals.csv"), - eurostat=input_eurostat, + eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition", pop_weighted_energy_totals=resources( "pop_weighted_energy_totals_s{simpl}_{clusters}.csv" ), diff --git a/rules/common.smk b/rules/common.smk index 618d746b..2b8495e1 100644 --- a/rules/common.smk +++ b/rules/common.smk @@ -130,12 +130,6 @@ def has_internet_access(url="www.zenodo.org") -> bool: conn.close() -def input_eurostat(w): - # 2016 includes BA, 2017 does not - report_year = config_provider("energy", "eurostat_report_year")(w) - return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition" - - def solved_previous_horizon(w): planning_horizons = config_provider("scenario", "planning_horizons")(w) i = planning_horizons.index(int(w.planning_horizons)) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index dc08699f..1b188829 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -237,7 +237,6 @@ rule plot_summary: countries=config_provider("countries"), planning_horizons=config_provider("scenario", "planning_horizons"), emissions_scope=config_provider("energy", "emissions"), - eurostat_report_year=config_provider("energy", "eurostat_report_year"), plotting=config_provider("plotting"), foresight=config_provider("foresight"), co2_budget=config_provider("co2_budget"), @@ -247,7 +246,7 @@ rule plot_summary: costs=RESULTS + "csvs/costs.csv", energy=RESULTS + "csvs/energy.csv", balances=RESULTS + "csvs/supply_energy.csv", - eurostat=input_eurostat, + eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition", co2="data/bundle-sector/eea/UNFCCC_v23.csv", output: costs=RESULTS + "graphs/costs.pdf", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 29d050ab..b516a0cd 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -135,20 +135,10 @@ if config["enable"]["retrieve"] and config["enable"].get( "h2_salt_caverns_GWh_per_sqkm.geojson", ] - datafolders = [ - protected( - directory("data/bundle-sector/eurostat-energy_balances-june_2016_edition") - ), - protected( - directory("data/bundle-sector/eurostat-energy_balances-may_2018_edition") - ), - protected(directory("data/bundle-sector/jrc-idees-2015")), - ] - rule retrieve_sector_databundle: output: protected(expand("data/bundle-sector/{files}", files=datafiles)), - *datafolders, + protected(directory("data/bundle-sector/jrc-idees-2015")), log: "logs/retrieve_sector_databundle.log", retries: 2 @@ -157,6 +147,15 @@ if config["enable"]["retrieve"] and config["enable"].get( script: "../scripts/retrieve_sector_databundle.py" + rule retrieve_eurostat_data: + output: + directory("data/eurostat/eurostat-energy_balances-april_2023_edition"), + log: + "logs/retrieve_eurostat_data.log", + retries: 2 + script: + "../scripts/retrieve_eurostat_data.py" + if config["enable"]["retrieve"]: datafiles = [ diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 44c9b20e..0bbe19f0 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -171,10 +171,6 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas phased_out = df_agg[df_agg["DateOut"] < baseyear].index df_agg.drop(phased_out, inplace=True) - # calculate remaining lifetime before phase-out (+1 because assuming - # phase out date at the end of the year) - df_agg["lifetime"] = df_agg.DateOut - df_agg.DateIn + 1 - # assign clustered bus busmap_s = pd.read_csv(snakemake.input.busmap_s, index_col=0).squeeze() busmap = pd.read_csv(snakemake.input.busmap, index_col=0).squeeze() @@ -195,6 +191,10 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas grouping_years, np.digitize(df_agg.DateIn, grouping_years, right=True) ) + # calculate (adjusted) remaining lifetime before phase-out (+1 because assuming + # phase out date at the end of the year) + df_agg["lifetime"] = df_agg.DateOut - df_agg["grouping_year"] + 1 + df = df_agg.pivot_table( index=["grouping_year", "Fueltype"], columns="cluster_bus", diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 960d1bbe..1ffc4ae2 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -8,6 +8,7 @@ Build total energy demands per country using JRC IDEES, eurostat, and EEA data. import logging import multiprocessing as mp +import os from functools import partial import country_converter as coco @@ -36,54 +37,6 @@ def reverse(dictionary): return {v: k for k, v in dictionary.items()} -eurostat_codes = { - "EU28": "EU", - "EA19": "EA", - "Belgium": "BE", - "Bulgaria": "BG", - "Czech Republic": "CZ", - "Denmark": "DK", - "Germany": "DE", - "Estonia": "EE", - "Ireland": "IE", - "Greece": "GR", - "Spain": "ES", - "France": "FR", - "Croatia": "HR", - "Italy": "IT", - "Cyprus": "CY", - "Latvia": "LV", - "Lithuania": "LT", - "Luxembourg": "LU", - "Hungary": "HU", - "Malta": "MA", - "Netherlands": "NL", - "Austria": "AT", - "Poland": "PL", - "Portugal": "PT", - "Romania": "RO", - "Slovenia": "SI", - "Slovakia": "SK", - "Finland": "FI", - "Sweden": "SE", - "United Kingdom": "GB", - "Iceland": "IS", - "Norway": "NO", - "Montenegro": "ME", - "FYR of Macedonia": "MK", - "Albania": "AL", - "Serbia": "RS", - "Turkey": "TU", - "Bosnia and Herzegovina": "BA", - "Kosovo\n(UNSCR 1244/99)": "KO", # 2017 version - # 2016 version - "Kosovo\n(under United Nations Security Council Resolution 1244/99)": "KO", - "Moldova": "MO", - "Ukraine": "UK", - "Switzerland": "CH", -} - - idees_rename = {"GR": "EL", "GB": "UK"} eu28 = cc.EU28as("ISO2").ISO2.tolist() @@ -116,39 +69,57 @@ to_ipcc = { } -def build_eurostat(input_eurostat, countries, report_year, year): +def build_eurostat(input_eurostat, countries, year): """ Return multi-index for all countries' energy data in TWh/a. """ - filenames = { - 2016: f"/{year}-Energy-Balances-June2016edition.xlsx", - 2017: f"/{year}-ENERGY-BALANCES-June2017edition.xlsx", - } - - with mute_print(): - dfs = pd.read_excel( - input_eurostat + filenames[report_year], - sheet_name=None, - skiprows=1, + df = {} + countries = {idees_rename.get(country, country) for country in countries} - {"CH"} + for country in countries: + filename = ( + f"{input_eurostat}/{country}-Energy-balance-sheets-April-2023-edition.xlsb" + ) + sheet = pd.read_excel( + filename, + engine="pyxlsb", + sheet_name=str(year), + skiprows=4, index_col=list(range(4)), ) + df[country] = sheet + df = pd.concat(df, axis=0) - # sorted_index necessary for slicing - lookup = eurostat_codes - labelled_dfs = { - lookup[df.columns[0]]: df - for df in dfs.values() - if lookup[df.columns[0]] in countries + # drop columns with all NaNs + unnamed_cols = df.columns[df.columns.astype(str).str.startswith("Unnamed")] + df.drop(unnamed_cols, axis=1, inplace=True) + df.drop(year, axis=1, inplace=True) + + # make numeric values where possible + df.replace("Z", 0, inplace=True) + df = df.apply(pd.to_numeric, errors="coerce") + df = df.select_dtypes(include=[np.number]) + + # write 'International aviation' to the 2nd level of the multiindex + int_avia = df.index.get_level_values(2) == "International aviation" + temp = df.loc[int_avia] + temp.index = pd.MultiIndex.from_frame( + temp.index.to_frame().fillna("International aviation") + ) + df = pd.concat([temp, df.loc[~int_avia]]) + + # Renaming some indices + index_rename = { + "Households": "Residential", + "Commercial & public services": "Services", + "Domestic navigation": "Domestic Navigation", + "International maritime bunkers": "Bunkers", } - df = pd.concat(labelled_dfs, sort=True).sort_index() + columns_rename = {"Total": "Total all products", "UK": "GB"} + df.rename(index=index_rename, columns=columns_rename, inplace=True) + df.sort_index(inplace=True) + df.index.names = [None] * len(df.index.names) - # drop non-numeric and country columns - non_numeric_cols = df.columns[df.dtypes != float] - country_cols = df.columns.intersection(lookup.keys()) - to_drop = non_numeric_cols.union(country_cols) - df.drop(to_drop, axis=1, inplace=True) - - # convert ktoe/a to TWh/a + # convert to TWh/a from ktoe/a df *= 11.63 / 1e3 return df @@ -651,8 +622,8 @@ def build_eea_co2(input_co2, year=1990, emissions_scope="CO2"): return emissions / 1e3 -def build_eurostat_co2(input_eurostat, countries, report_year, year=1990): - eurostat = build_eurostat(input_eurostat, countries, report_year, year) +def build_eurostat_co2(input_eurostat, countries, year=1990): + eurostat = build_eurostat(input_eurostat, countries, year) specific_emissions = pd.Series(index=eurostat.columns, dtype=float) @@ -674,12 +645,7 @@ def build_co2_totals(countries, eea_co2, eurostat_co2): for ct in pd.Index(countries).intersection(["BA", "RS", "AL", "ME", "MK"]): mappings = { - "electricity": ( - ct, - "+", - "Conventional Thermal Power Stations", - "of which From Coal", - ), + "electricity": (ct, "+", "Electricity & heat generation", np.nan), "residential non-elec": (ct, "+", "+", "Residential"), "services non-elec": (ct, "+", "+", "Services"), "road non-elec": (ct, "+", "+", "Road"), @@ -687,12 +653,12 @@ def build_co2_totals(countries, eea_co2, eurostat_co2): "domestic navigation": (ct, "+", "+", "Domestic Navigation"), "international navigation": (ct, "-", "Bunkers"), "domestic aviation": (ct, "+", "+", "Domestic aviation"), - "international aviation": (ct, "+", "+", "International aviation"), + "international aviation": (ct, "-", "International aviation"), # does not include industrial process emissions or fuel processing/refining - "industrial non-elec": (ct, "+", "Industry"), + "industrial non-elec": (ct, "+", "Industry sector"), # does not include non-energy emissions "agriculture": (eurostat_co2.index.get_level_values(0) == ct) - & eurostat_co2.index.isin(["Agriculture / Forestry", "Fishing"], level=3), + & eurostat_co2.index.isin(["Agriculture & forestry", "Fishing"], level=3), } for i, mi in mappings.items(): @@ -737,6 +703,131 @@ def build_transport_data(countries, population, idees): return transport_data +def rescale_idees_from_eurostat( + idees_countries, energy, eurostat, input_eurostat, countries +): + """ + Takes JRC IDEES data from 2015 and rescales it by the ratio of the eurostat + data and the 2015 eurostat data. + + missing data: ['passenger car efficiency', 'passenger cars'] + """ + main_cols = ["Total all products", "Electricity"] + # read in the eurostat data for 2015 + eurostat_2015 = build_eurostat(input_eurostat, countries, 2015)[main_cols] + eurostat_year = eurostat[main_cols] + # calculate the ratio of the two data sets + ratio = eurostat_year / eurostat_2015 + ratio = ratio.droplevel([1, 4]) + cols_rename = {"Total all products": "total", "Electricity": "ele"} + index_rename = {v: k for k, v in idees_rename.items()} + ratio.rename(columns=cols_rename, index=index_rename, inplace=True) + + mappings = { + "Residential": { + "total": [ + "total residential space", + "total residential water", + "total residential cooking", + "total residential", + "derived heat residential", + "thermal uses residential", + ], + "elec": [ + "electricity residential space", + "electricity residential water", + "electricity residential cooking", + "electricity residential", + ], + }, + "Services": { + "total": [ + "total services space", + "total services water", + "total services cooking", + "total services", + "derived heat services", + "thermal uses services", + ], + "elec": [ + "electricity services space", + "electricity services water", + "electricity services cooking", + "electricity services", + ], + }, + "Agriculture & forestry": { + "total": [ + "total agriculture heat", + "total agriculture machinery", + "total agriculture", + ], + "elec": [ + "total agriculture electricity", + ], + }, + "Road": { + "total": [ + "total road", + "total passenger cars", + "total other road passenger", + "total light duty road freight", + ], + "elec": [ + "electricity road", + "electricity passenger cars", + "electricity other road passenger", + "electricity light duty road freight", + ], + }, + "Rail": { + "total": [ + "total rail", + "total rail passenger", + "total rail freight", + ], + "elec": [ + "electricity rail", + "electricity rail passenger", + "electricity rail freight", + ], + }, + } + + avia_inter = [ + "total aviation passenger", + "total aviation freight", + "total international aviation passenger", + "total international aviation freight", + "total international aviation", + ] + avia_domestic = [ + "total domestic aviation passenger", + "total domestic aviation freight", + "total domestic aviation", + ] + navigation = [ + "total domestic navigation", + ] + + for country in idees_countries: + for sector, mapping in mappings.items(): + sector_ratio = ratio.loc[(country, slice(None), sector)] + + energy.loc[country, mapping["total"]] *= sector_ratio["total"].iloc[0] + energy.loc[country, mapping["elec"]] *= sector_ratio["ele"].iloc[0] + + avi_d = ratio.loc[(country, slice(None), "Domestic aviation"), "total"] + avi_i = ratio.loc[(country, "International aviation", slice(None)), "total"] + energy.loc[country, avia_inter] *= avi_i.iloc[0] + energy.loc[country, avia_domestic] *= avi_d.iloc[0] + + nav = ratio.loc[(country, slice(None), "Domestic Navigation"), "total"] + energy.loc[country, navigation] *= nav.iloc[0] + + return energy + + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -755,24 +846,32 @@ if __name__ == "__main__": idees_countries = pd.Index(countries).intersection(eu28) data_year = params["energy_totals_year"] - report_year = snakemake.params.energy["eurostat_report_year"] input_eurostat = snakemake.input.eurostat - eurostat = build_eurostat(input_eurostat, countries, report_year, data_year) + eurostat = build_eurostat(input_eurostat, countries, data_year) swiss = build_swiss(data_year) - idees = build_idees(idees_countries, data_year) + # data from idees only exists from 2000-2015. read in latest data and rescale later + idees = build_idees(idees_countries, min(2015, data_year)) energy = build_energy_totals(countries, eurostat, swiss, idees) + + if data_year > 2015: + logger.info("Data year is after 2015. Rescaling IDEES data based on eurostat.") + energy = rescale_idees_from_eurostat( + idees_countries, energy, eurostat, input_eurostat, countries + ) + energy.to_csv(snakemake.output.energy_name) - district_heat_share = build_district_heat_share(countries, idees) + # use rescaled idees data to calculate district heat share + district_heat_share = build_district_heat_share( + countries, energy.loc[idees_countries] + ) district_heat_share.to_csv(snakemake.output.district_heat_share) base_year_emissions = params["base_emissions_year"] emissions_scope = snakemake.params.energy["emissions"] eea_co2 = build_eea_co2(snakemake.input.co2, base_year_emissions, emissions_scope) - eurostat_co2 = build_eurostat_co2( - input_eurostat, countries, report_year, base_year_emissions - ) + eurostat_co2 = build_eurostat_co2(input_eurostat, countries, base_year_emissions) co2 = build_co2_totals(countries, eea_co2, eurostat_co2) co2.to_csv(snakemake.output.co2_name) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 2ad37d3f..5c14b065 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -97,33 +97,18 @@ fields = { "Other Industrial Sectors": "Physical output (index)", } -eb_names = { - "NO": "Norway", - "AL": "Albania", - "BA": "Bosnia and Herzegovina", - "MK": "FYR of Macedonia", - "GE": "Georgia", - "IS": "Iceland", - "KO": "Kosovo", - "MD": "Moldova", - "ME": "Montenegro", - "RS": "Serbia", - "UA": "Ukraine", - "TR": "Turkey", -} - eb_sectors = { - "Iron & steel industry": "Iron and steel", - "Chemical and Petrochemical industry": "Chemicals Industry", - "Non-ferrous metal industry": "Non-metallic mineral products", - "Paper, Pulp and Print": "Pulp, paper and printing", - "Food and Tabacco": "Food, beverages and tobacco", - "Non-metallic Minerals (Glass, pottery & building mat. Industry)": "Non Ferrous Metals", - "Transport Equipment": "Transport Equipment", + "Iron & steel": "Iron and steel", + "Chemical & petrochemical": "Chemicals Industry", + "Non-ferrous metals": "Non-metallic mineral products", + "Paper, pulp & printing": "Pulp, paper and printing", + "Food, beverages & tobacco": "Food, beverages and tobacco", + "Non-metallic minerals": "Non Ferrous Metals", + "Transport equipment": "Transport Equipment", "Machinery": "Machinery Equipment", - "Textile and Leather": "Textiles and leather", - "Wood and Wood Products": "Wood and wood products", - "Non-specified (Industry)": "Other Industrial Sectors", + "Textile & leather": "Textiles and leather", + "Wood & wood products": "Wood and wood products", + "Not elsewhere specified (industry)": "Other Industrial Sectors", } # TODO: this should go in a csv in `data` @@ -160,12 +145,15 @@ def get_energy_ratio(country, eurostat_dir, jrc_dir, year): e_country = e_switzerland * tj_to_ktoe else: # estimate physical output, energy consumption in the sector and country - fn = f"{eurostat_dir}/{eb_names[country]}.XLSX" - with mute_print(): - df = pd.read_excel( - fn, sheet_name="2016", index_col=2, header=0, skiprows=1 - ).squeeze("columns") - e_country = df.loc[eb_sectors.keys(), "Total all products"].rename(eb_sectors) + fn = f"{eurostat_dir}/{country}-Energy-balance-sheets-April-2023-edition.xlsb" + df = pd.read_excel( + fn, + sheet_name=str(min(2021, year)), + index_col=2, + header=0, + skiprows=4, + ) + e_country = df.loc[eb_sectors.keys(), "Total"].rename(eb_sectors) fn = f"{jrc_dir}/JRC-IDEES-2015_Industry_EU28.xlsx" diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index c2fd7e04..bfe9995f 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -462,7 +462,6 @@ def plot_carbon_budget_distribution(input_eurostat, options): plt.rcParams["ytick.labelsize"] = 20 emissions_scope = snakemake.params.emissions_scope - report_year = snakemake.params.eurostat_report_year input_co2 = snakemake.input.co2 # historic emissions @@ -472,7 +471,6 @@ def plot_carbon_budget_distribution(input_eurostat, options): input_eurostat, options, emissions_scope, - report_year, input_co2, year=1990, ) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index bb4b71a8..71772e81 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -249,7 +249,7 @@ def get(item, investment_year=None): def co2_emissions_year( - countries, input_eurostat, options, emissions_scope, report_year, input_co2, year + countries, input_eurostat, options, emissions_scope, input_co2, year ): """ Calculate CO2 emissions in one specific year (e.g. 1990 or 2018). @@ -259,11 +259,9 @@ def co2_emissions_year( # TODO: read Eurostat data from year > 2014 # this only affects the estimation of CO2 emissions for BA, RS, AL, ME, MK if year > 2014: - eurostat_co2 = build_eurostat_co2( - input_eurostat, countries, report_year, year=2014 - ) + eurostat_co2 = build_eurostat_co2(input_eurostat, countries, 2014) else: - eurostat_co2 = build_eurostat_co2(input_eurostat, countries, report_year, year) + eurostat_co2 = build_eurostat_co2(input_eurostat, countries, year) co2_totals = build_co2_totals(countries, eea_co2, eurostat_co2) @@ -278,9 +276,7 @@ def co2_emissions_year( # TODO: move to own rule with sector-opts wildcard? -def build_carbon_budget( - o, input_eurostat, fn, emissions_scope, report_year, input_co2, options -): +def build_carbon_budget(o, input_eurostat, fn, emissions_scope, input_co2, options): """ Distribute carbon budget following beta or exponential transition path. """ @@ -301,7 +297,6 @@ def build_carbon_budget( input_eurostat, options, emissions_scope, - report_year, input_co2, year=1990, ) @@ -312,7 +307,6 @@ def build_carbon_budget( input_eurostat, options, emissions_scope, - report_year, input_co2, year=2018, ) @@ -3823,14 +3817,12 @@ if __name__ == "__main__": fn = "results/" + snakemake.params.RDIR + "/csvs/carbon_budget_distribution.csv" if not os.path.exists(fn): emissions_scope = snakemake.params.emissions_scope - report_year = snakemake.params.eurostat_report_year input_co2 = snakemake.input.co2 build_carbon_budget( co2_budget, snakemake.input.eurostat, fn, emissions_scope, - report_year, input_co2, options, ) diff --git a/scripts/retrieve_eurostat_data.py b/scripts/retrieve_eurostat_data.py new file mode 100644 index 00000000..4b4cea4a --- /dev/null +++ b/scripts/retrieve_eurostat_data.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2024- The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT +""" +Retrieve and extract eurostat energy balances data. +""" + + +import logging +import zipfile +from pathlib import Path + +from _helpers import configure_logging, progress_retrieve, set_scenario_config + +logger = logging.getLogger(__name__) + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake("retrieve_eurostat_data") + rootpath = ".." + else: + rootpath = "." + configure_logging(snakemake) + set_scenario_config(snakemake) + + disable_progress = snakemake.config["run"].get("disable_progressbar", False) + url_eurostat = "https://ec.europa.eu/eurostat/documents/38154/4956218/Balances-December2022.zip/f7cf0d19-5c0f-60ad-4e48-098a5ddd6e48?t=1671184070589" + tarball_fn = Path(f"{rootpath}/data/eurostat/eurostat_2023.zip") + to_fn = Path( + f"{rootpath}/data/eurostat/eurostat-energy_balances-april_2023_edition/" + ) + + logger.info(f"Downloading Eurostat data from '{url_eurostat}'.") + progress_retrieve(url_eurostat, tarball_fn, disable=disable_progress) + + logger.info("Extracting Eurostat data.") + with zipfile.ZipFile(tarball_fn, "r") as zip_ref: + zip_ref.extractall(to_fn) + + logger.info(f"Eurostat data available in '{to_fn}'.")