Merge branch 'master' into multiyear
This commit is contained in:
Fabian Neumann
commit
579cd0c756
2
.github/workflows/ci.yaml
vendored
2
.github/workflows/ci.yaml
vendored
@ -19,7 +19,7 @@ on:
|
||||
- cron: "0 5 * * TUE"
|
||||
|
||||
env:
|
||||
DATA_CACHE_NUMBER: 2
|
||||
DATA_CACHE_NUMBER: 1
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
||||
@ -319,9 +319,8 @@ pypsa_eur:
|
||||
|
||||
# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#energy
|
||||
energy:
|
||||
energy_totals_year: 2013
|
||||
energy_totals_year: 2019
|
||||
base_emissions_year: 1990
|
||||
eurostat_report_year: 2016
|
||||
emissions: CO2
|
||||
|
||||
# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#biomass
|
||||
@ -360,7 +359,7 @@ solar_thermal:
|
||||
|
||||
# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#existing-capacities
|
||||
existing_capacities:
|
||||
grouping_years_power: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030]
|
||||
grouping_years_power: [1960, 1965, 1970, 1975, 1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030]
|
||||
grouping_years_heat: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019] # these should not extend 2020
|
||||
threshold_capacity: 10
|
||||
default_heating_lifetime: 20
|
||||
|
||||
@ -1,25 +1,25 @@
|
||||
country,item,2010,2011,2012,2013,2014,2015
|
||||
CH,total residential,268.2,223.4,243.4,261.3,214.2,229.1
|
||||
CH,total residential space,192.2,149.0,168.1,185.5,139.7,154.4
|
||||
CH,total residential water,32.2,31.6,31.9,32.2,31.7,31.9
|
||||
CH,total residential cooking,9.3,9.3,9.3,9.4,9.5,9.6
|
||||
CH,electricity residential,67.9,63.7,65.7,67.6,63.0,64.4
|
||||
CH,electricity residential space,15.9,12.8,14.3,15.8,12.3,13.5
|
||||
CH,electricity residential water,8.8,8.5,8.5,8.6,8.5,8.6
|
||||
CH,electricity residential cooking,4.9,4.9,4.9,4.9,5.0,5.0
|
||||
CH,total services,145.9,127.4,136.7,144.0,124.5,132.5
|
||||
CH,total services space,80.0,62.2,70.8,77.4,58.3,64.3
|
||||
CH,total services water,10.1,10.0,10.1,10.1,10.0,10.0
|
||||
CH,total services cooking,2.5,2.4,2.3,2.3,2.4,2.3
|
||||
CH,electricity services,60.5,59.2,60.3,61.4,60.3,62.6
|
||||
CH,electricity services space,4.0,3.2,3.8,4.2,3.3,3.6
|
||||
CH,electricity services water,0.7,0.7,0.7,0.7,0.7,0.7
|
||||
CH,electricity services cooking,2.5,2.4,2.3,2.3,2.4,2.3
|
||||
CH,total rail,11.5,11.1,11.2,11.4,11.1,11.4
|
||||
CH,total road,199.4,200.4,200.4,201.2,202.0,203.1
|
||||
CH,electricity road,0.,0.,0.,0.,0.,0.
|
||||
CH,electricity rail,11.5,11.1,11.2,11.4,11.1,11.4
|
||||
CH,total domestic aviation,3.3,3.2,3.4,3.4,3.5,3.5
|
||||
CH,total international aviation,58.0,62.0,63.5,64.2,64.5,66.8
|
||||
CH,total domestic navigation,1.6,1.6,1.6,1.6,1.6,1.6
|
||||
CH,total international navigation,0.,0.,0.,0.,0.,0.
|
||||
country,item,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022
|
||||
CH,total residential,268.2,223.4,243.4,261.3,214.2,229.1,241.2,236.5,223.7,226.5,219.1,241.2,211.3
|
||||
CH,total residential space,192.2,149,168.1,185.5,139.7,154.4,167.3,161.5,147.2,150.4,140.2,166.2,131.9
|
||||
CH,total residential water,32.2,31.6,31.9,32.2,31.7,31.9,31.8,31.8,31.8,31.7,33.3,32.5,32.5
|
||||
CH,total residential cooking,9.3,9.3,9.3,9.4,9.5,9.6,9.9,10,10.1,10.2,10.5,10.3,10.3
|
||||
CH,electricity residential,67.9,63.7,65.7,67.6,63,64.4,69.7,69.2,67.7,68.1,68.7,70.8,66.8
|
||||
CH,electricity residential space,15.9,12.8,14.3,15.8,12.3,13.5,15.8,15.6,14.7,15.3,14.8,17.8,14.8
|
||||
CH,electricity residential water,8.8,8.5,8.5,8.6,8.5,8.6,8.9,9,9.2,9.3,9.7,9.5,9.5
|
||||
CH,electricity residential cooking,4.9,4.9,4.9,4.9,5,5,5,5.1,5.1,5.1,5.4,5.2,5.3
|
||||
CH,total services,145.9,127.4,136.7,144,124.5,132.5,150.5,147.7,141.5,143.1,129.7,144.2,122.5
|
||||
CH,total services space,80,62.2,70.8,77.4,58.3,64.3,77,74.4,68.2,69.8,64.3,75.7,58.7
|
||||
CH,total services water,10.1,10,10.1,10.1,10,10,11.4,11.3,11.2,11.1,9.7,10.4,12
|
||||
CH,total services cooking,2.5,2.4,2.3,2.3,2.4,2.3,3.1,3.1,3.2,3.3,2.1,2.6,3.2
|
||||
CH,electricity services,60.5,59.2,60.3,61.4,60.3,62.6,65.9,65.7,65.5,65.6,58.8,61.6,61.6
|
||||
CH,electricity services space,4,3.2,3.8,4.2,3.3,3.6,2.7,2.5,2.3,2.3,2.2,2.5,2.5
|
||||
CH,electricity services water,0.7,0.7,0.7,0.7,0.7,0.7,1.2,1.1,1.1,1.1,0.9,1,1
|
||||
CH,electricity services cooking,2.5,2.4,2.3,2.3,2.4,2.3,3.1,3.1,3.1,3.2,3.3,2.1,3.2
|
||||
CH,total rail,11.5,11.1,11.2,11.4,11.1,11.4,11.6,11.4,11.2,11,10.2,10.6,10.8
|
||||
CH,total road,199.4,200.4,200.4,201.2,202,203.1,203.9,203.7,202.6,200.5,182.6,188.3,193.3
|
||||
CH,electricity road,0,0,0,0,0,0,0.1,0.2,0.3,0.4,0.5,0.8,1.3
|
||||
CH,electricity rail,11.5,11.1,11.2,11.4,11.1,11.4,11.5,11.3,11.1,11,10.1,10.6,10.7
|
||||
CH,total domestic aviation,3.3,3.2,3.4,3.4,3.5,3.5,3.6,3.1,3.1,2.9,2.5,2.8,3
|
||||
CH,total international aviation,58,62,63.5,64.2,64.5,66.8,70.6,72.8,77.2,78.2,28.2,31.2,56.8
|
||||
CH,total domestic navigation,1.6,1.6,1.6,1.6,1.6,1.6,1.4,1.4,1.4,1.4,1.4,1.4,1.4
|
||||
CH,total international navigation,0,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
|
||||
|
@ -1,7 +1,4 @@
|
||||
,Unit,Values,Description
|
||||
energy_totals_year ,--,"{1990,1995,2000,2005,2010,2011,…} ",The year for the sector energy use. The year must be avaliable in the Eurostat report
|
||||
base_emissions_year ,--,"YYYY; e.g. 1990","The base year for the sector emissions. See `European Environment Agency (EEA) <https://www.eea.europa.eu/data-and-maps/data/national-emissions-reported-to-the-unfccc-and-to-the-eu-greenhouse-gas-monitoring-mechanism-16>`_."
|
||||
|
||||
eurostat_report_year ,--,"{2016,2017,2018}","The publication year of the Eurostat report. 2016 includes Bosnia and Herzegovina, 2017 does not"
|
||||
|
||||
emissions ,--,"{CO2, All greenhouse gases - (CO2 equivalent)}","Specify which sectoral emissions are taken into account. Data derived from EEA. Currently only CO2 is implemented."
|
||||
|
||||
|
@ -9,6 +9,23 @@ Release Notes
|
||||
|
||||
Upcoming Release
|
||||
================
|
||||
|
||||
* Corrected a bug leading to power plants operating after their DateOut
|
||||
(https://github.com/PyPSA/pypsa-eur/pull/958). Added additional grouping years
|
||||
before 1980.
|
||||
|
||||
* The Eurostat data was updated to the 2023 version in :mod:`build_energy_totals`.
|
||||
|
||||
* The latest `Swiss energy totals
|
||||
<https://www.bfe.admin.ch/bfe/de/home/versorgung/statistik-und-geodaten/energiestatistiken/energieverbrauch-nach-verwendungszweck.html/>`_
|
||||
have been updated to the 2023 version.
|
||||
|
||||
* The JRC-IDEES data is only available until 2015. For energy totals years (``energy: energy_totals_year``) after
|
||||
2015, the data scaled using the ratio of Eurostat data reported for the energy
|
||||
totals year and 2015.
|
||||
|
||||
* The default energy totals year (``energy: energy_totals_year``) was updated to 2019.
|
||||
|
||||
* Upgrade default techno-economic assumptions to ``technology-data`` v0.8.1.
|
||||
|
||||
* Linearly interpolate missing investment periods in year-dependent
|
||||
|
||||
@ -291,7 +291,7 @@ rule build_energy_totals:
|
||||
swiss="data/switzerland-new_format-all_years.csv",
|
||||
idees="data/bundle-sector/jrc-idees-2015",
|
||||
district_heat_share="data/district_heat_share.csv",
|
||||
eurostat="data/eurostat-energy_balances-june_2021_edition",
|
||||
eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition",
|
||||
output:
|
||||
energy_name=resources("energy_totals.csv"),
|
||||
co2_name=resources("co2_totals.csv"),
|
||||
@ -508,7 +508,7 @@ rule build_industrial_production_per_country:
|
||||
input:
|
||||
ammonia_production=resources("ammonia_production.csv"),
|
||||
jrc="data/bundle-sector/jrc-idees-2015",
|
||||
eurostat="data/bundle-sector/eurostat-energy_balances-may_2018_edition",
|
||||
eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition",
|
||||
output:
|
||||
industrial_production_per_country=resources(
|
||||
"industrial_production_per_country.csv"
|
||||
@ -878,7 +878,6 @@ rule prepare_sector_network:
|
||||
countries=config_provider("countries"),
|
||||
adjustments=config_provider("adjustments", "sector"),
|
||||
emissions_scope=config_provider("energy", "emissions"),
|
||||
eurostat_report_year=config_provider("energy", "eurostat_report_year"),
|
||||
RDIR=RDIR,
|
||||
input:
|
||||
unpack(input_profile_offwind),
|
||||
@ -909,7 +908,7 @@ rule prepare_sector_network:
|
||||
),
|
||||
network=resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"),
|
||||
energy_totals_name=resources("energy_totals.csv"),
|
||||
eurostat=input_eurostat,
|
||||
eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition",
|
||||
pop_weighted_energy_totals=resources(
|
||||
"pop_weighted_energy_totals_s{simpl}_{clusters}.csv"
|
||||
),
|
||||
|
||||
@ -130,12 +130,6 @@ def has_internet_access(url="www.zenodo.org") -> bool:
|
||||
conn.close()
|
||||
|
||||
|
||||
def input_eurostat(w):
|
||||
# 2016 includes BA, 2017 does not
|
||||
report_year = config_provider("energy", "eurostat_report_year")(w)
|
||||
return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition"
|
||||
|
||||
|
||||
def solved_previous_horizon(w):
|
||||
planning_horizons = config_provider("scenario", "planning_horizons")(w)
|
||||
i = planning_horizons.index(int(w.planning_horizons))
|
||||
|
||||
@ -237,7 +237,6 @@ rule plot_summary:
|
||||
countries=config_provider("countries"),
|
||||
planning_horizons=config_provider("scenario", "planning_horizons"),
|
||||
emissions_scope=config_provider("energy", "emissions"),
|
||||
eurostat_report_year=config_provider("energy", "eurostat_report_year"),
|
||||
plotting=config_provider("plotting"),
|
||||
foresight=config_provider("foresight"),
|
||||
co2_budget=config_provider("co2_budget"),
|
||||
@ -247,7 +246,7 @@ rule plot_summary:
|
||||
costs=RESULTS + "csvs/costs.csv",
|
||||
energy=RESULTS + "csvs/energy.csv",
|
||||
balances=RESULTS + "csvs/supply_energy.csv",
|
||||
eurostat=input_eurostat,
|
||||
eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition",
|
||||
co2="data/bundle-sector/eea/UNFCCC_v23.csv",
|
||||
output:
|
||||
costs=RESULTS + "graphs/costs.pdf",
|
||||
|
||||
@ -135,21 +135,10 @@ if config["enable"]["retrieve"] and config["enable"].get(
|
||||
"h2_salt_caverns_GWh_per_sqkm.geojson",
|
||||
]
|
||||
|
||||
# TODO: check which versions of eurostat to keep
|
||||
datafolders = [
|
||||
protected(
|
||||
directory("data/bundle-sector/eurostat-energy_balances-june_2016_edition")
|
||||
),
|
||||
protected(
|
||||
directory("data/bundle-sector/eurostat-energy_balances-may_2018_edition")
|
||||
),
|
||||
protected(directory("data/bundle-sector/jrc-idees-2015")),
|
||||
]
|
||||
|
||||
rule retrieve_sector_databundle:
|
||||
output:
|
||||
protected(expand("data/bundle-sector/{files}", files=datafiles)),
|
||||
*datafolders,
|
||||
protected(directory("data/bundle-sector/jrc-idees-2015")),
|
||||
log:
|
||||
"logs/retrieve_sector_databundle.log",
|
||||
retries: 2
|
||||
@ -158,6 +147,15 @@ if config["enable"]["retrieve"] and config["enable"].get(
|
||||
script:
|
||||
"../scripts/retrieve_sector_databundle.py"
|
||||
|
||||
rule retrieve_eurostat_data:
|
||||
output:
|
||||
directory("data/eurostat/eurostat-energy_balances-april_2023_edition"),
|
||||
log:
|
||||
"logs/retrieve_eurostat_data.log",
|
||||
retries: 2
|
||||
script:
|
||||
"../scripts/retrieve_eurostat_data.py"
|
||||
|
||||
|
||||
if config["enable"]["retrieve"]:
|
||||
datafiles = [
|
||||
|
||||
@ -171,10 +171,6 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas
|
||||
phased_out = df_agg[df_agg["DateOut"] < baseyear].index
|
||||
df_agg.drop(phased_out, inplace=True)
|
||||
|
||||
# calculate remaining lifetime before phase-out (+1 because assuming
|
||||
# phase out date at the end of the year)
|
||||
df_agg["lifetime"] = df_agg.DateOut - df_agg.DateIn + 1
|
||||
|
||||
# assign clustered bus
|
||||
busmap_s = pd.read_csv(snakemake.input.busmap_s, index_col=0).squeeze()
|
||||
busmap = pd.read_csv(snakemake.input.busmap, index_col=0).squeeze()
|
||||
@ -195,6 +191,10 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas
|
||||
grouping_years, np.digitize(df_agg.DateIn, grouping_years, right=True)
|
||||
)
|
||||
|
||||
# calculate (adjusted) remaining lifetime before phase-out (+1 because assuming
|
||||
# phase out date at the end of the year)
|
||||
df_agg["lifetime"] = df_agg.DateOut - df_agg["grouping_year"] + 1
|
||||
|
||||
df = df_agg.pivot_table(
|
||||
index=["grouping_year", "Fueltype"],
|
||||
columns="cluster_bus",
|
||||
|
||||
@ -8,6 +8,7 @@ Build total energy demands per country using JRC IDEES, eurostat, and EEA data.
|
||||
|
||||
import logging
|
||||
import multiprocessing as mp
|
||||
import os
|
||||
from functools import partial
|
||||
|
||||
import country_converter as coco
|
||||
@ -36,8 +37,6 @@ def reverse(dictionary):
|
||||
return {v: k for k, v in dictionary.items()}
|
||||
|
||||
|
||||
non_EU = ["NO", "CH", "ME", "MK", "RS", "BA", "AL"]
|
||||
|
||||
idees_rename = {"GR": "EL", "GB": "UK"}
|
||||
|
||||
eu28 = cc.EU28as("ISO2").ISO2.tolist()
|
||||
@ -70,60 +69,57 @@ to_ipcc = {
|
||||
}
|
||||
|
||||
|
||||
def eurostat_per_country(country):
|
||||
|
||||
country_fn = idees_rename.get(country, country)
|
||||
fn = (
|
||||
snakemake.input.eurostat
|
||||
+ f"/{country_fn}-Energy-balance-sheets-June-2021-edition.xlsb"
|
||||
)
|
||||
|
||||
df = pd.read_excel(
|
||||
fn,
|
||||
sheet_name=None,
|
||||
skiprows=4,
|
||||
index_col=list(range(3)),
|
||||
na_values=["+", "-", "=", "Z", ":"],
|
||||
)
|
||||
|
||||
df.pop("Cover")
|
||||
|
||||
return pd.concat(df)
|
||||
|
||||
|
||||
def build_eurostat(countries, year=None):
|
||||
def build_eurostat(input_eurostat, countries, year):
|
||||
"""
|
||||
Return multi-index for all countries' energy data in TWh/a.
|
||||
"""
|
||||
df = {}
|
||||
countries = {idees_rename.get(country, country) for country in countries} - {"CH"}
|
||||
for country in countries:
|
||||
filename = (
|
||||
f"{input_eurostat}/{country}-Energy-balance-sheets-April-2023-edition.xlsb"
|
||||
)
|
||||
sheet = pd.read_excel(
|
||||
filename,
|
||||
engine="pyxlsb",
|
||||
sheet_name=str(year),
|
||||
skiprows=4,
|
||||
index_col=list(range(4)),
|
||||
)
|
||||
df[country] = sheet
|
||||
df = pd.concat(df, axis=0)
|
||||
|
||||
nprocesses = snakemake.threads
|
||||
tqdm_kwargs = dict(
|
||||
ascii=False,
|
||||
unit=" country",
|
||||
total=len(countries),
|
||||
desc="Build from eurostat database",
|
||||
# drop columns with all NaNs
|
||||
unnamed_cols = df.columns[df.columns.astype(str).str.startswith("Unnamed")]
|
||||
df.drop(unnamed_cols, axis=1, inplace=True)
|
||||
df.drop(year, axis=1, inplace=True)
|
||||
|
||||
# make numeric values where possible
|
||||
df.replace("Z", 0, inplace=True)
|
||||
df = df.apply(pd.to_numeric, errors="coerce")
|
||||
df = df.select_dtypes(include=[np.number])
|
||||
|
||||
# write 'International aviation' to the 2nd level of the multiindex
|
||||
int_avia = df.index.get_level_values(2) == "International aviation"
|
||||
temp = df.loc[int_avia]
|
||||
temp.index = pd.MultiIndex.from_frame(
|
||||
temp.index.to_frame().fillna("International aviation")
|
||||
)
|
||||
with mp.Pool(processes=nprocesses) as pool:
|
||||
dfs = list(tqdm(pool.imap(eurostat_per_country, countries), **tqdm_kwargs))
|
||||
df = pd.concat([temp, df.loc[~int_avia]])
|
||||
|
||||
index_names = ["country", "year", "lvl1", "lvl2", "lvl3"]
|
||||
df = pd.concat(dfs, keys=countries, names=index_names)
|
||||
# Renaming some indices
|
||||
index_rename = {
|
||||
"Households": "Residential",
|
||||
"Commercial & public services": "Services",
|
||||
"Domestic navigation": "Domestic Navigation",
|
||||
"International maritime bunkers": "Bunkers",
|
||||
}
|
||||
columns_rename = {"Total": "Total all products", "UK": "GB"}
|
||||
df.rename(index=index_rename, columns=columns_rename, inplace=True)
|
||||
df.sort_index(inplace=True)
|
||||
df.index.names = [None] * len(df.index.names)
|
||||
|
||||
df.dropna(how="all", axis=0, inplace=True)
|
||||
df.dropna(how="all", axis=1, inplace=True)
|
||||
df = df[df.index.get_level_values("lvl1") != "ktoe"]
|
||||
|
||||
i = df.index.to_frame(index=False)
|
||||
i.loc[i.lvl2 == "Primary production", ["lvl1", "lvl3"]] = "Main"
|
||||
i.loc[i.lvl2 == "Gross electricity production", "lvl1"] = "Gross production"
|
||||
i.ffill(inplace=True)
|
||||
df.index = pd.MultiIndex.from_frame(i)
|
||||
|
||||
df.drop(list(range(1990, 2020)), axis=1, inplace=True)
|
||||
df.drop("Unnamed: 7", axis=1, inplace=True)
|
||||
df.fillna(0.0, inplace=True)
|
||||
|
||||
# convert ktoe/a to TWh/a
|
||||
# convert to TWh/a from ktoe/a
|
||||
df *= 11.63 / 1e3
|
||||
|
||||
df.index = df.index.set_levels(df.index.levels[1].astype(int), level=1)
|
||||
@ -659,7 +655,8 @@ def build_eea_co2(input_co2, year=1990, emissions_scope="CO2"):
|
||||
return emissions / 1e3
|
||||
|
||||
|
||||
def build_eurostat_co2(countries, eurostat=None, year=1990):
|
||||
def build_eurostat_co2(input_eurostat, countries, year=1990):
|
||||
eurostat = build_eurostat(input_eurostat, countries, year)
|
||||
|
||||
if eurostat is None:
|
||||
df = build_eurostat(countries, year)
|
||||
@ -688,57 +685,17 @@ def build_co2_totals(countries, eea_co2, eurostat_co2):
|
||||
|
||||
for ct in pd.Index(countries).intersection(["BA", "RS", "AL", "ME", "MK"]):
|
||||
mappings = {
|
||||
"electricity": (
|
||||
ct,
|
||||
"Transformation input",
|
||||
"Electricity & heat generation",
|
||||
"Main",
|
||||
),
|
||||
"residential non-elec": (
|
||||
ct,
|
||||
"Final energy consumption",
|
||||
"Other sectors",
|
||||
"Households",
|
||||
),
|
||||
"services non-elec": (
|
||||
ct,
|
||||
"Final energy consumption",
|
||||
"Other sectors",
|
||||
"Commercial & public services",
|
||||
),
|
||||
"road non-elec": (
|
||||
ct,
|
||||
"Final energy consumption",
|
||||
"Transport sector",
|
||||
"Road",
|
||||
),
|
||||
"rail non-elec": (
|
||||
ct,
|
||||
"Final energy consumption",
|
||||
"Transport sector",
|
||||
"Rail",
|
||||
),
|
||||
"domestic navigation": (
|
||||
ct,
|
||||
"Final energy consumption",
|
||||
"Transport sector",
|
||||
"Domestic navigation",
|
||||
),
|
||||
"international navigation": (ct, "Main", "International maritime bunkers"),
|
||||
"domestic aviation": (
|
||||
ct,
|
||||
"Final energy consumption",
|
||||
"Transport sector",
|
||||
"Domestic aviation",
|
||||
),
|
||||
"international aviation": (ct, "Main", "International aviation"),
|
||||
"electricity": (ct, "+", "Electricity & heat generation", np.nan),
|
||||
"residential non-elec": (ct, "+", "+", "Residential"),
|
||||
"services non-elec": (ct, "+", "+", "Services"),
|
||||
"road non-elec": (ct, "+", "+", "Road"),
|
||||
"rail non-elec": (ct, "+", "+", "Rail"),
|
||||
"domestic navigation": (ct, "+", "+", "Domestic Navigation"),
|
||||
"international navigation": (ct, "-", "Bunkers"),
|
||||
"domestic aviation": (ct, "+", "+", "Domestic aviation"),
|
||||
"international aviation": (ct, "-", "International aviation"),
|
||||
# does not include industrial process emissions or fuel processing/refining
|
||||
"industrial non-elec": (
|
||||
ct,
|
||||
"Final energy consumption",
|
||||
"Industry sector",
|
||||
"Non-energy use in industry sector",
|
||||
),
|
||||
"industrial non-elec": (ct, "+", "Industry sector"),
|
||||
# does not include non-energy emissions
|
||||
"agriculture": (eurostat_co2.index.get_level_values(0) == ct)
|
||||
& eurostat_co2.index.isin(["Agriculture & forestry", "Fishing"], level=3),
|
||||
@ -786,6 +743,131 @@ def build_transport_data(countries, population, idees):
|
||||
return transport_data
|
||||
|
||||
|
||||
def rescale_idees_from_eurostat(
|
||||
idees_countries, energy, eurostat, input_eurostat, countries
|
||||
):
|
||||
"""
|
||||
Takes JRC IDEES data from 2015 and rescales it by the ratio of the eurostat
|
||||
data and the 2015 eurostat data.
|
||||
|
||||
missing data: ['passenger car efficiency', 'passenger cars']
|
||||
"""
|
||||
main_cols = ["Total all products", "Electricity"]
|
||||
# read in the eurostat data for 2015
|
||||
eurostat_2015 = build_eurostat(input_eurostat, countries, 2015)[main_cols]
|
||||
eurostat_year = eurostat[main_cols]
|
||||
# calculate the ratio of the two data sets
|
||||
ratio = eurostat_year / eurostat_2015
|
||||
ratio = ratio.droplevel([1, 4])
|
||||
cols_rename = {"Total all products": "total", "Electricity": "ele"}
|
||||
index_rename = {v: k for k, v in idees_rename.items()}
|
||||
ratio.rename(columns=cols_rename, index=index_rename, inplace=True)
|
||||
|
||||
mappings = {
|
||||
"Residential": {
|
||||
"total": [
|
||||
"total residential space",
|
||||
"total residential water",
|
||||
"total residential cooking",
|
||||
"total residential",
|
||||
"derived heat residential",
|
||||
"thermal uses residential",
|
||||
],
|
||||
"elec": [
|
||||
"electricity residential space",
|
||||
"electricity residential water",
|
||||
"electricity residential cooking",
|
||||
"electricity residential",
|
||||
],
|
||||
},
|
||||
"Services": {
|
||||
"total": [
|
||||
"total services space",
|
||||
"total services water",
|
||||
"total services cooking",
|
||||
"total services",
|
||||
"derived heat services",
|
||||
"thermal uses services",
|
||||
],
|
||||
"elec": [
|
||||
"electricity services space",
|
||||
"electricity services water",
|
||||
"electricity services cooking",
|
||||
"electricity services",
|
||||
],
|
||||
},
|
||||
"Agriculture & forestry": {
|
||||
"total": [
|
||||
"total agriculture heat",
|
||||
"total agriculture machinery",
|
||||
"total agriculture",
|
||||
],
|
||||
"elec": [
|
||||
"total agriculture electricity",
|
||||
],
|
||||
},
|
||||
"Road": {
|
||||
"total": [
|
||||
"total road",
|
||||
"total passenger cars",
|
||||
"total other road passenger",
|
||||
"total light duty road freight",
|
||||
],
|
||||
"elec": [
|
||||
"electricity road",
|
||||
"electricity passenger cars",
|
||||
"electricity other road passenger",
|
||||
"electricity light duty road freight",
|
||||
],
|
||||
},
|
||||
"Rail": {
|
||||
"total": [
|
||||
"total rail",
|
||||
"total rail passenger",
|
||||
"total rail freight",
|
||||
],
|
||||
"elec": [
|
||||
"electricity rail",
|
||||
"electricity rail passenger",
|
||||
"electricity rail freight",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
avia_inter = [
|
||||
"total aviation passenger",
|
||||
"total aviation freight",
|
||||
"total international aviation passenger",
|
||||
"total international aviation freight",
|
||||
"total international aviation",
|
||||
]
|
||||
avia_domestic = [
|
||||
"total domestic aviation passenger",
|
||||
"total domestic aviation freight",
|
||||
"total domestic aviation",
|
||||
]
|
||||
navigation = [
|
||||
"total domestic navigation",
|
||||
]
|
||||
|
||||
for country in idees_countries:
|
||||
for sector, mapping in mappings.items():
|
||||
sector_ratio = ratio.loc[(country, slice(None), sector)]
|
||||
|
||||
energy.loc[country, mapping["total"]] *= sector_ratio["total"].iloc[0]
|
||||
energy.loc[country, mapping["elec"]] *= sector_ratio["ele"].iloc[0]
|
||||
|
||||
avi_d = ratio.loc[(country, slice(None), "Domestic aviation"), "total"]
|
||||
avi_i = ratio.loc[(country, "International aviation", slice(None)), "total"]
|
||||
energy.loc[country, avia_inter] *= avi_i.iloc[0]
|
||||
energy.loc[country, avia_domestic] *= avi_d.iloc[0]
|
||||
|
||||
nav = ratio.loc[(country, slice(None), "Domestic Navigation"), "total"]
|
||||
energy.loc[country, navigation] *= nav.iloc[0]
|
||||
|
||||
return energy
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if "snakemake" not in globals():
|
||||
from _helpers import mock_snakemake
|
||||
@ -805,22 +887,32 @@ if __name__ == "__main__":
|
||||
countries_without_ch = pd.Index(countries).difference(["CH"])
|
||||
|
||||
data_year = params["energy_totals_year"]
|
||||
report_year = snakemake.params.energy["eurostat_report_year"]
|
||||
input_eurostat = snakemake.input.eurostat
|
||||
eurostat = build_eurostat(countries_without_ch)
|
||||
swiss = build_swiss()
|
||||
idees = build_idees(idees_countries, data_year)
|
||||
eurostat = build_eurostat(input_eurostat, countries, data_year)
|
||||
swiss = build_swiss(data_year)
|
||||
# data from idees only exists from 2000-2015. read in latest data and rescale later
|
||||
idees = build_idees(idees_countries, min(2015, data_year))
|
||||
|
||||
energy = build_energy_totals(countries, eurostat, swiss, idees)
|
||||
|
||||
if data_year > 2015:
|
||||
logger.info("Data year is after 2015. Rescaling IDEES data based on eurostat.")
|
||||
energy = rescale_idees_from_eurostat(
|
||||
idees_countries, energy, eurostat, input_eurostat, countries
|
||||
)
|
||||
|
||||
energy.to_csv(snakemake.output.energy_name)
|
||||
|
||||
district_heat_share = build_district_heat_share(countries, idees)
|
||||
# use rescaled idees data to calculate district heat share
|
||||
district_heat_share = build_district_heat_share(
|
||||
countries, energy.loc[idees_countries]
|
||||
)
|
||||
district_heat_share.to_csv(snakemake.output.district_heat_share)
|
||||
|
||||
base_year_emissions = params["base_emissions_year"]
|
||||
emissions_scope = snakemake.params.energy["emissions"]
|
||||
eea_co2 = build_eea_co2(snakemake.input.co2, base_year_emissions, emissions_scope)
|
||||
eurostat_co2 = build_eurostat_co2(countries, eurostat, base_year_emissions)
|
||||
eurostat_co2 = build_eurostat_co2(input_eurostat, countries, base_year_emissions)
|
||||
|
||||
co2 = build_co2_totals(countries, eea_co2, eurostat_co2)
|
||||
co2.to_csv(snakemake.output.co2_name)
|
||||
|
||||
@ -97,33 +97,18 @@ fields = {
|
||||
"Other Industrial Sectors": "Physical output (index)",
|
||||
}
|
||||
|
||||
eb_names = {
|
||||
"NO": "Norway",
|
||||
"AL": "Albania",
|
||||
"BA": "Bosnia and Herzegovina",
|
||||
"MK": "FYR of Macedonia",
|
||||
"GE": "Georgia",
|
||||
"IS": "Iceland",
|
||||
"KO": "Kosovo",
|
||||
"MD": "Moldova",
|
||||
"ME": "Montenegro",
|
||||
"RS": "Serbia",
|
||||
"UA": "Ukraine",
|
||||
"TR": "Turkey",
|
||||
}
|
||||
|
||||
eb_sectors = {
|
||||
"Iron & steel industry": "Iron and steel",
|
||||
"Chemical and Petrochemical industry": "Chemicals Industry",
|
||||
"Non-ferrous metal industry": "Non-metallic mineral products",
|
||||
"Paper, Pulp and Print": "Pulp, paper and printing",
|
||||
"Food and Tabacco": "Food, beverages and tobacco",
|
||||
"Non-metallic Minerals (Glass, pottery & building mat. Industry)": "Non Ferrous Metals",
|
||||
"Transport Equipment": "Transport Equipment",
|
||||
"Iron & steel": "Iron and steel",
|
||||
"Chemical & petrochemical": "Chemicals Industry",
|
||||
"Non-ferrous metals": "Non-metallic mineral products",
|
||||
"Paper, pulp & printing": "Pulp, paper and printing",
|
||||
"Food, beverages & tobacco": "Food, beverages and tobacco",
|
||||
"Non-metallic minerals": "Non Ferrous Metals",
|
||||
"Transport equipment": "Transport Equipment",
|
||||
"Machinery": "Machinery Equipment",
|
||||
"Textile and Leather": "Textiles and leather",
|
||||
"Wood and Wood Products": "Wood and wood products",
|
||||
"Non-specified (Industry)": "Other Industrial Sectors",
|
||||
"Textile & leather": "Textiles and leather",
|
||||
"Wood & wood products": "Wood and wood products",
|
||||
"Not elsewhere specified (industry)": "Other Industrial Sectors",
|
||||
}
|
||||
|
||||
# TODO: this should go in a csv in `data`
|
||||
@ -160,12 +145,15 @@ def get_energy_ratio(country, eurostat_dir, jrc_dir, year):
|
||||
e_country = e_switzerland * tj_to_ktoe
|
||||
else:
|
||||
# estimate physical output, energy consumption in the sector and country
|
||||
fn = f"{eurostat_dir}/{eb_names[country]}.XLSX"
|
||||
with mute_print():
|
||||
df = pd.read_excel(
|
||||
fn, sheet_name="2016", index_col=2, header=0, skiprows=1
|
||||
).squeeze("columns")
|
||||
e_country = df.loc[eb_sectors.keys(), "Total all products"].rename(eb_sectors)
|
||||
fn = f"{eurostat_dir}/{country}-Energy-balance-sheets-April-2023-edition.xlsb"
|
||||
df = pd.read_excel(
|
||||
fn,
|
||||
sheet_name=str(min(2021, year)),
|
||||
index_col=2,
|
||||
header=0,
|
||||
skiprows=4,
|
||||
)
|
||||
e_country = df.loc[eb_sectors.keys(), "Total"].rename(eb_sectors)
|
||||
|
||||
fn = f"{jrc_dir}/JRC-IDEES-2015_Industry_EU28.xlsx"
|
||||
|
||||
|
||||
@ -462,7 +462,6 @@ def plot_carbon_budget_distribution(input_eurostat, options):
|
||||
plt.rcParams["ytick.labelsize"] = 20
|
||||
|
||||
emissions_scope = snakemake.params.emissions_scope
|
||||
report_year = snakemake.params.eurostat_report_year
|
||||
input_co2 = snakemake.input.co2
|
||||
|
||||
# historic emissions
|
||||
@ -472,7 +471,6 @@ def plot_carbon_budget_distribution(input_eurostat, options):
|
||||
input_eurostat,
|
||||
options,
|
||||
emissions_scope,
|
||||
report_year,
|
||||
input_co2,
|
||||
year=1990,
|
||||
)
|
||||
|
||||
@ -248,7 +248,7 @@ def get(item, investment_year=None):
|
||||
|
||||
|
||||
def co2_emissions_year(
|
||||
countries, input_eurostat, options, emissions_scope, report_year, input_co2, year
|
||||
countries, input_eurostat, options, emissions_scope, input_co2, year
|
||||
):
|
||||
"""
|
||||
Calculate CO2 emissions in one specific year (e.g. 1990 or 2018).
|
||||
@ -258,11 +258,9 @@ def co2_emissions_year(
|
||||
# TODO: read Eurostat data from year > 2014
|
||||
# this only affects the estimation of CO2 emissions for BA, RS, AL, ME, MK
|
||||
if year > 2014:
|
||||
eurostat_co2 = build_eurostat_co2(
|
||||
input_eurostat, countries, report_year, year=2014
|
||||
)
|
||||
eurostat_co2 = build_eurostat_co2(input_eurostat, countries, 2014)
|
||||
else:
|
||||
eurostat_co2 = build_eurostat_co2(input_eurostat, countries, report_year, year)
|
||||
eurostat_co2 = build_eurostat_co2(input_eurostat, countries, year)
|
||||
|
||||
co2_totals = build_co2_totals(countries, eea_co2, eurostat_co2)
|
||||
|
||||
@ -277,9 +275,7 @@ def co2_emissions_year(
|
||||
|
||||
|
||||
# TODO: move to own rule with sector-opts wildcard?
|
||||
def build_carbon_budget(
|
||||
o, input_eurostat, fn, emissions_scope, report_year, input_co2, options
|
||||
):
|
||||
def build_carbon_budget(o, input_eurostat, fn, emissions_scope, input_co2, options):
|
||||
"""
|
||||
Distribute carbon budget following beta or exponential transition path.
|
||||
"""
|
||||
@ -300,7 +296,6 @@ def build_carbon_budget(
|
||||
input_eurostat,
|
||||
options,
|
||||
emissions_scope,
|
||||
report_year,
|
||||
input_co2,
|
||||
year=1990,
|
||||
)
|
||||
@ -311,7 +306,6 @@ def build_carbon_budget(
|
||||
input_eurostat,
|
||||
options,
|
||||
emissions_scope,
|
||||
report_year,
|
||||
input_co2,
|
||||
year=2018,
|
||||
)
|
||||
@ -3678,14 +3672,12 @@ if __name__ == "__main__":
|
||||
fn = "results/" + snakemake.params.RDIR + "/csvs/carbon_budget_distribution.csv"
|
||||
if not os.path.exists(fn):
|
||||
emissions_scope = snakemake.params.emissions_scope
|
||||
report_year = snakemake.params.eurostat_report_year
|
||||
input_co2 = snakemake.input.co2
|
||||
build_carbon_budget(
|
||||
co2_budget,
|
||||
snakemake.input.eurostat,
|
||||
fn,
|
||||
emissions_scope,
|
||||
report_year,
|
||||
input_co2,
|
||||
options,
|
||||
)
|
||||
|
||||
43
scripts/retrieve_eurostat_data.py
Normal file
43
scripts/retrieve_eurostat_data.py
Normal file
@ -0,0 +1,43 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# SPDX-FileCopyrightText: : 2024- The PyPSA-Eur Authors
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
"""
|
||||
Retrieve and extract eurostat energy balances data.
|
||||
"""
|
||||
|
||||
|
||||
import logging
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
from _helpers import configure_logging, progress_retrieve, set_scenario_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
if __name__ == "__main__":
|
||||
if "snakemake" not in globals():
|
||||
from _helpers import mock_snakemake
|
||||
|
||||
snakemake = mock_snakemake("retrieve_eurostat_data")
|
||||
rootpath = ".."
|
||||
else:
|
||||
rootpath = "."
|
||||
configure_logging(snakemake)
|
||||
set_scenario_config(snakemake)
|
||||
|
||||
disable_progress = snakemake.config["run"].get("disable_progressbar", False)
|
||||
url_eurostat = "https://ec.europa.eu/eurostat/documents/38154/4956218/Balances-December2022.zip/f7cf0d19-5c0f-60ad-4e48-098a5ddd6e48?t=1671184070589"
|
||||
tarball_fn = Path(f"{rootpath}/data/eurostat/eurostat_2023.zip")
|
||||
to_fn = Path(
|
||||
f"{rootpath}/data/eurostat/eurostat-energy_balances-april_2023_edition/"
|
||||
)
|
||||
|
||||
logger.info(f"Downloading Eurostat data from '{url_eurostat}'.")
|
||||
progress_retrieve(url_eurostat, tarball_fn, disable=disable_progress)
|
||||
|
||||
logger.info("Extracting Eurostat data.")
|
||||
with zipfile.ZipFile(tarball_fn, "r") as zip_ref:
|
||||
zip_ref.extractall(to_fn)
|
||||
|
||||
logger.info(f"Eurostat data available in '{to_fn}'.")
|
||||
Loading…
Reference in New Issue
Block a user