import pandas as pd
import geopandas as gpd
idx = pd.IndexSlice
#translations for Eurostat
country_to_code = {
'EU28' : 'EU',
'EA19' : 'EA',
'Belgium' : 'BE',
'Bulgaria' : 'BG',
'Czech Republic' : 'CZ',
'Denmark' : 'DK',
'Germany' : 'DE',
'Estonia' : 'EE',
'Ireland' : 'IE',
'Greece' : 'GR',
'Spain' : 'ES',
'France' : 'FR',
'Croatia' : 'HR',
'Italy' : 'IT',
'Cyprus' : 'CY',
'Latvia' : 'LV',
'Lithuania' : 'LT',
'Luxembourg' : 'LU',
'Hungary' : 'HU',
'Malta' : 'MA',
'Netherlands' : 'NL',
'Austria' : 'AT',
'Poland' : 'PL',
'Portugal' : 'PT',
'Romania' : 'RO',
'Slovenia' : 'SI',
'Slovakia' : 'SK',
'Finland' : 'FI',
'Sweden' : 'SE',
'United Kingdom' : 'GB',
'Iceland' : 'IS',
'Norway' : 'NO',
'Montenegro' : 'ME',
'FYR of Macedonia' : 'MK',
'Albania' : 'AL',
'Serbia' : 'RS',
'Turkey' : 'TU',
'Bosnia and Herzegovina' : 'BA',
'Kosovo\n(UNSCR 1244/99)' : 'KO', #2017 version
'Kosovo\n(under United Nations Security Council Resolution 1244/99)' : 'KO', #2016 version
'Moldova' : 'MO',
'Ukraine' : 'UK',
'Switzerland' : 'CH',
}
non_EU = ['NO', 'CH', 'ME', 'MK', 'RS', 'BA', 'AL']
rename = {"GR" : "EL",
"GB" : "UK"}
eu28 = ['FR', 'DE', 'GB', 'IT', 'ES', 'PL', 'SE', 'NL', 'BE', 'FI', 'CZ',
'DK', 'PT', 'RO', 'AT', 'BG', 'EE', 'GR', 'LV',
'HU', 'IE', 'SK', 'LT', 'HR', 'LU', 'SI'] + ['CY','MT']
eu28_eea = eu28[:]
eu28_eea.remove("GB")
eu28_eea.append("UK")
def build_eurostat(year):
"""Return multi-index for all countries' energy data in TWh/a."""
stats_from_year = 2016
fns = {2016: "data/eurostat-energy_balances-june_2016_edition/{year}-Energy-Balances-June2016edition.xlsx",
2017: "data/eurostat-energy_balances-june_2017_edition/{year}-ENERGY-BALANCES-June2017edition.xlsx"}
#2016 includes BA, 2017 doesn't
#with sheet as None, an ordered dictionary of all sheets is returned
dfs = pd.read_excel(fns[stats_from_year].format(year=year),
None,
skiprows=1,
index_col=list(range(4)))
#sorted_index necessary for slicing
df = pd.concat({country_to_code[df.columns[0]] : df for ct,df in dfs.items()},sort=True).sort_index()
#drop non-numeric columns; convert ktoe/a to TWh/a
return df.drop(df.columns[df.dtypes != float],axis=1)*11.63/1e3
def build_swiss(year):
fn = "data/switzerland-sfoe/switzerland-new_format.csv"
#convert PJ/a to TWh/a
return (pd.read_csv(fn,index_col=list(range(2)))/3.6).loc["CH",str(year)]
def build_idees(year):
base_dir = "data/jrc-idees-2015"
totals = pd.DataFrame()
#convert ktoe/a to TWh/a
factor = 11.63/1e3
for ct in population.index:
if ct in non_EU:
print("When reading IDEES, skipping non-EU28 country",ct)
continue
#RESIDENTIAL
filename = "{}/JRC-IDEES-2015_Residential_{}.xlsx".format(base_dir,rename.get(ct,ct))
df = pd.read_excel(filename,"RES_hh_fec")
assert df.iloc[2,0] == "Space heating"
totals.loc[ct,"total residential space"] = df.loc[2,year]
assert df.iloc[10,0] == "Advanced electric heating"
assert df.iloc[11,0] == "Conventional electric heating"
totals.loc[ct,"electricity residential space"] = df.loc[[10,11],year].sum()
assert df.iloc[15,0] == "Water heating"
totals.loc[ct,"total residential water"] = df.loc[15,year]
assert df.iloc[23,0] == "Electricity"
totals.loc[ct,"electricity residential water"] = df.loc[23,year]
assert df.iloc[25,0] == "Cooking"
totals.loc[ct,"total residential cooking"] = df.loc[25,year]
assert df.iloc[30,0] == "Electricity"
totals.loc[ct,"electricity residential cooking"] = df.loc[30,year]
df = pd.read_excel(filename,"RES_summary")
assert df.iloc[34,0] == "Energy consumption by fuel - Eurostat structure (ktoe)"
totals.loc[ct,"total residential"] = df.loc[34,year]
assert df.iloc[47,0] == "Electricity"
totals.loc[ct,"electricity residential"] = df.loc[47,year]
#SERVICES
filename = "{}/JRC-IDEES-2015_Tertiary_{}.xlsx".format(base_dir,rename.get(ct,ct))
df = pd.read_excel(filename,"SER_hh_fec")
assert df.iloc[2,0] == "Space heating"
totals.loc[ct,"total services space"] = df.loc[2,year]
assert df.iloc[11,0] == "Advanced electric heating"
assert df.iloc[12,0] == "Conventional electric heating"
totals.loc[ct,"electricity services space"] = df.loc[[11,12],year].sum()
assert df.iloc[17,0] == "Hot water"
totals.loc[ct,"total services water"] = df.loc[17,year]
assert df.iloc[24,0] == "Electricity"
totals.loc[ct,"electricity services water"] = df.loc[24,year]
assert df.iloc[27,0] == "Catering"
totals.loc[ct,"total services cooking"] = df.loc[27,year]
assert df.iloc[31,0] == "Electricity"
totals.loc[ct,"electricity services cooking"] = df.loc[31,year]
df = pd.read_excel(filename,"SER_summary")
assert df.iloc[37,0] == "Energy consumption by fuel - Eurostat structure (ktoe)"
totals.loc[ct,"total services"] = df.loc[37,year]
assert df.iloc[50,0] == "Electricity"
totals.loc[ct,"electricity services"] = df.loc[50,year]
# TRANSPORT
filename = "{}/JRC-IDEES-2015_Transport_{}.xlsx".format(base_dir,rename.get(ct,ct))
df = pd.read_excel(filename,"TrRoad_ene")
assert df.iloc[2,0] == "by fuel (EUROSTAT DATA)"
totals.loc[ct,"total road"] = df.loc[2,year]
assert df.iloc[13,0] == "Electricity"
totals.loc[ct,"electricity road"] = df.loc[13,year]
assert df.iloc[17,0] == "Powered 2-wheelers (Gasoline)"
totals.loc[ct,"total two-wheel"] = df.loc[17,year]
assert df.iloc[19,0] == "Passenger cars"
totals.loc[ct,"total passenger cars"] = df.loc[19,year]
assert df.iloc[30,0] == "Battery electric vehicles"
totals.loc[ct,"electricity passenger cars"] = df.loc[30,year]
assert df.iloc[31,0] == "Motor coaches, buses and trolley buses"
totals.loc[ct,"total other road passenger"] = df.loc[31,year]
assert df.iloc[39,0] == "Battery electric vehicles"
totals.loc[ct,"electricity other road passenger"] = df.loc[39,year]
assert df.iloc[41,0] == "Light duty vehicles"
totals.loc[ct,"total light duty road freight"] = df.loc[41,year]
assert df.iloc[49,0] == "Battery electric vehicles"
totals.loc[ct,"electricity light duty road freight"] = df.loc[49,year]
assert df.iloc[50,0] == "Heavy duty vehicles (Diesel oil incl. biofuels)"
totals.loc[ct,"total heavy duty road freight"] = df.loc[50,year]
assert df.iloc[61,0] == "Passenger cars"
totals.loc[ct,"passenger car efficiency"] = df.loc[61,year]
df = pd.read_excel(filename,"TrRail_ene")
assert df.iloc[2,0] == "by fuel (EUROSTAT DATA)"
totals.loc[ct,"total rail"] = df.loc[2,year]
assert df.iloc[12,0] == "Electricity"
totals.loc[ct,"electricity rail"] = df.loc[12,year]
assert df.iloc[15,0] == "Passenger transport"
totals.loc[ct,"total rail passenger"] = df.loc[15,year]
assert df.iloc[16,0] == "Metro and tram, urban light rail"
assert df.iloc[19,0] == "Electric"
assert df.iloc[20,0] == "High speed passenger trains"
totals.loc[ct,"electricity rail passenger"] = df.loc[[16,19,20],year].sum()
assert df.iloc[21,0] == "Freight transport"
totals.loc[ct,"total rail freight"] = df.loc[21,year]
assert df.iloc[23,0] == "Electric"
totals.loc[ct,"electricity rail freight"] = df.loc[23,year]
df = pd.read_excel(filename,"TrAvia_ene")
assert df.iloc[6,0] == "Passenger transport"
totals.loc[ct,"total aviation passenger"] = df.loc[6,year]
assert df.iloc[10,0] == "Freight transport"
totals.loc[ct,"total aviation freight"] = df.loc[10,year]
assert df.iloc[7,0] == "Domestic"
totals.loc[ct,"total domestic aviation passenger"] = df.loc[7,year]
assert df.iloc[8,0] == "International - Intra-EU"
assert df.iloc[9,0] == "International - Extra-EU"
totals.loc[ct,"total international aviation passenger"] = df.loc[[8,9],year].sum()
assert df.iloc[11,0] == "Domestic and International - Intra-EU"
totals.loc[ct,"total domestic aviation freight"] = df.loc[11,year]
assert df.iloc[12,0] == "International - Extra-EU"
totals.loc[ct,"total international aviation freight"] = df.loc[12,year]
totals.loc[ct,"total domestic aviation"] = totals.loc[ct,["total domestic aviation freight","total domestic aviation passenger"]].sum()
totals.loc[ct,"total international aviation"] = totals.loc[ct,["total international aviation freight","total international aviation passenger"]].sum()
df = pd.read_excel(filename,"TrNavi_ene")
#coastal and inland
assert df.iloc[2,0] == "by fuel (EUROSTAT DATA)"
totals.loc[ct,"total domestic navigation"] = df.loc[2,year]
df = pd.read_excel(filename,"TrRoad_act")
assert df.iloc[85,0] == "Passenger cars"
totals.loc[ct,"passenger cars"] = df.loc[85,year]
totals = totals*factor
totals["passenger cars"] = totals["passenger cars"]/factor
#convert ktoe/100km to kWh per km
totals["passenger car efficiency"] = 10*totals["passenger car efficiency"]
return totals
def build_energy_totals():
clean_df = idees.reindex(population.index).drop(["passenger cars","passenger car efficiency"],axis=1)
print("International navigation")
in_eurostat = clean_df.index.intersection(eurostat.index.levels[0])
clean_df.loc[in_eurostat,"total international navigation"] = eurostat.loc[idx[in_eurostat,:,"Bunkers",:],"Total all products"].groupby(level=0).sum()
clean_df.loc["CH"] = swiss
#get values for missing countries based on Eurostat EnergyBalances
#divide cooking/space/water according to averages in EU28
missing = clean_df.index[clean_df["total residential"].isnull()]
missing_in_eurostat = missing.intersection(eurostat.index.levels[0])
uses = ["space","cooking","water"]
for sector,eurostat_sector in [("residential","Residential"),("services","Services"),
("road","Road"),("rail","Rail")]:
for fuel,eurostat_fuel in [("electricity","Electricity"),("total","Total all products")]:
clean_df.loc[missing_in_eurostat,"{} {}".format(fuel,sector)] = eurostat.loc[idx[missing_in_eurostat,:,:,eurostat_sector],eurostat_fuel].groupby(level=0).sum()
if sector in ["road","rail"]:
continue
fuel = "electricity"
for use in uses:
avg = (clean_df["{} {} {}".format(fuel,sector,use)]/clean_df["{} {}".format(fuel,sector)]).mean()
print("{}: average fraction of {} for {} is {}".format(sector,fuel,use,avg))
clean_df.loc[missing_in_eurostat,"{} {} {}".format(fuel,sector,use)] = avg*clean_df.loc[missing_in_eurostat,"{} {}".format(fuel,sector)]
fuel = "total"
for use in uses:
avg = ((clean_df["{} {} {}".format("total",sector,use)]-clean_df["{} {} {}".format("electricity",sector,use)])/
(clean_df["{} {}".format("total",sector)]-clean_df["{} {}".format("electricity",sector)])).mean()
print("{}: average fraction of non-electric for {} is {}".format(sector,use,avg))
clean_df.loc[missing_in_eurostat,"{} {} {}".format(fuel,sector,use)] = \
clean_df.loc[missing_in_eurostat,"{} {} {}".format("electricity",sector,use)] \
+ avg*(clean_df.loc[missing_in_eurostat,"{} {}".format("total",sector)] - clean_df.loc[missing_in_eurostat,"{} {}".format("electricity",sector)])
#Fix Norway space and water heating fractions
#http://www.ssb.no/en/energi-og-industri/statistikker/husenergi/hvert-3-aar/2014-07-14
#The main heating source for about 73 per cent of the households is based on electricity
#=> 26% is non-electric
elec_fraction = 0.73
without_norway = clean_df.drop("NO")
for sector in ["residential","services"]:
#assume non-electric is heating
total_heating = (clean_df.loc["NO","{} {}".format("total",sector)]-clean_df.loc["NO","{} {}".format("electricity",sector)])/(1-elec_fraction)
for use in uses:
fraction = ((without_norway["{} {} {}".format("total",sector,use)]-without_norway["{} {} {}".format("electricity",sector,use)])/
(without_norway["{} {}".format("total",sector)]-without_norway["{} {}".format("electricity",sector)])).mean()
clean_df.loc["NO","{} {} {}".format("total",sector,use)] = total_heating*fraction
clean_df.loc["NO","{} {} {}".format("electricity",sector,use)] = total_heating*fraction*elec_fraction
#Missing aviation
print("Aviation")
clean_df.loc[missing_in_eurostat,"total domestic aviation"] = eurostat.loc[idx[missing_in_eurostat,:,:,"Domestic aviation"],"Total all products"].groupby(level=0).sum()
clean_df.loc[missing_in_eurostat,"total international aviation"] = eurostat.loc[idx[missing_in_eurostat,:,:,"International aviation"],"Total all products"].groupby(level=0).sum()
print("Domestic navigation")
clean_df.loc[missing_in_eurostat,"total domestic navigation"] = eurostat.loc[idx[missing_in_eurostat,:,:,"Domestic Navigation"],"Total all products"].groupby(level=0).sum()
#split road traffic for non-IDEES
missing = clean_df.index[clean_df["total passenger cars"].isnull()]
for fuel in ["total","electricity"]:
selection = [fuel+" passenger cars",fuel+" other road passenger",fuel+" light duty road freight"]
if fuel == "total":
selection = [fuel+" two-wheel"] + selection + [fuel+" heavy duty road freight"]
road = clean_df[selection].sum()
road_fraction = road/road.sum()
for i in road_fraction.index:
clean_df.loc[missing,i] = road_fraction[i]*clean_df.loc[missing,fuel+" road"]
#split rail traffic for non-IDEES
missing = clean_df.index[clean_df["total rail passenger"].isnull()]
for fuel in ["total","electricity"]:
selection = [fuel+" rail passenger",fuel+" rail freight"]
rail = clean_df[selection].sum()
rail_fraction = rail/rail.sum()
for i in rail_fraction.index:
clean_df.loc[missing,i] = rail_fraction[i]*clean_df.loc[missing,fuel+" rail"].values
#split aviation traffic for non-IDEES
missing = clean_df.index[clean_df["total domestic aviation passenger"].isnull()]
for destination in ["domestic","international"]:
selection = ["total " + destination+" aviation passenger","total " + destination+" aviation freight"]
aviation = clean_df[selection].sum()
aviation_fraction = aviation/aviation.sum()
for i in aviation_fraction.index:
clean_df.loc[missing,i] = aviation_fraction[i]*clean_df.loc[missing,"total "+ destination + " aviation"].values
clean_df.loc[missing,"total aviation passenger"] = clean_df.loc[missing,["total domestic aviation passenger","total international aviation passenger"]].sum(axis=1)
clean_df.loc[missing,"total aviation freight"] = clean_df.loc[missing,["total domestic aviation freight","total international aviation freight"]].sum(axis=1)
if "BA" in clean_df.index:
#fix missing data for BA (services and road energy data)
missing = (clean_df.loc["BA"] == 0.)
#add back in proportional to RS with ratio of total residential demand
clean_df.loc["BA",missing] = clean_df.loc["BA","total residential"]/clean_df.loc["RS","total residential"]*clean_df.loc["RS",missing]
clean_df.to_csv(snakemake.output.energy_name)
return clean_df
def build_eea_co2(year=1990):
# see ../notebooks/compute_1990_Europe_emissions_for_targets.ipynb
#https://www.eea.europa.eu/data-and-maps/data/national-emissions-reported-to-the-unfccc-and-to-the-eu-greenhouse-gas-monitoring-mechanism-16
#downloaded 201228 (modified by EEA last on 201221)
fn = "data/eea/UNFCCC_v23.csv"
df = pd.read_csv(fn, encoding="latin-1")
df.loc[df["Year"] == "1985-1987","Year"] = 1986
df["Year"] = df["Year"].astype(int)
df = df.set_index(['Country_code', 'Pollutant_name', 'Year', 'Sector_name']).sort_index()
e = pd.Series()
e["electricity"] = '1.A.1.a - Public Electricity and Heat Production'
e['residential non-elec'] = '1.A.4.b - Residential'
e['services non-elec'] = '1.A.4.a - Commercial/Institutional'
e['rail non-elec'] = "1.A.3.c - Railways"
e["road non-elec"] = '1.A.3.b - Road Transportation'
e["domestic navigation"] = "1.A.3.d - Domestic Navigation"
e['international navigation'] = '1.D.1.b - International Navigation'
e["domestic aviation"] = '1.A.3.a - Domestic Aviation'
e["international aviation"] = '1.D.1.a - International Aviation'
e['total energy'] = '1 - Energy'
e['industrial processes'] = '2 - Industrial Processes and Product Use'
e['agriculture'] = '3 - Agriculture'
e['LULUCF'] = '4 - Land Use, Land-Use Change and Forestry'
e['waste management'] = '5 - Waste management'
e['other'] = '6 - Other Sector'
e['indirect'] = 'ind_CO2 - Indirect CO2'
e["total wL"] = "Total (with LULUCF)"
e["total woL"] = "Total (without LULUCF)"
pol = "CO2" #["All greenhouse gases - (CO2 equivalent)","CO2"]
cts = ["CH","EUA","NO"] + eu28_eea
emissions = df.loc[idx[cts,pol,year,e.values],"emissions"].unstack("Sector_name").rename(columns=pd.Series(e.index,e.values)).rename(index={"All greenhouse gases - (CO2 equivalent)" : "GHG"},level=1)
#only take level 0, since level 1 (pol) and level 2 (year) are trivial
emissions = emissions.groupby(level=0,axis=0).sum()
emissions.rename(index={"EUA" : "EU28", "UK" : "GB"},inplace=True)
emissions['industrial non-elec'] = emissions['total energy'] - emissions[['electricity', 'services non-elec','residential non-elec', 'road non-elec',
'rail non-elec', 'domestic aviation', 'international aviation', 'domestic navigation',
'international navigation']].sum(axis=1)
emissions.drop(columns=["total energy", "total wL", "total woL"],inplace=True)
return emissions/1e3
def build_eurostat_co2(year=1990):
eurostat_for_co2 = build_eurostat(year)
se = pd.Series(index=eurostat_for_co2.columns,dtype=float)
#emissions in tCO2_equiv per MWh_th
se["Solid fuels"] = 0.36 #Approximates coal
se["Oil (total)"] = 0.285 #Average of distillate and residue
se["Gas"] = 0.2 #For natural gas
#oil values from https://www.eia.gov/tools/faqs/faq.cfm?id=74&t=11
#Distillate oil (No. 2) 0.276
#Residual oil (No. 6) 0.298
#https://www.eia.gov/electricity/annual/html/epa_a_03.html
eurostat_co2 = eurostat_for_co2.multiply(se).sum(axis=1)
return eurostat_co2
def build_co2_totals(eea_co2, eurostat_co2, year=1990):
co2 = eea_co2.reindex(["EU28","NO","CH","BA","RS","AL","ME","MK"] + eu28)
for ct in ["BA","RS","AL","ME","MK"]:
co2.loc[ct,"electricity"] = eurostat_co2[ct,"+","Conventional Thermal Power Stations","of which From Coal"].sum()
co2.loc[ct,"residential non-elec"] = eurostat_co2[ct,"+","+","Residential"].sum()
co2.loc[ct,"services non-elec"] = eurostat_co2[ct,"+","+","Services"].sum()
co2.loc[ct,"road non-elec"] = eurostat_co2[ct,"+","+","Road"].sum()
co2.loc[ct,"rail non-elec"] = eurostat_co2[ct,"+","+","Rail"].sum()
co2.loc[ct,"domestic navigation"] = eurostat_co2[ct,"+","+","Domestic Navigation"].sum()
co2.loc[ct,'international navigation'] = eurostat_co2[ct,"-","Bunkers"].sum()
co2.loc[ct,"domestic aviation"] = eurostat_co2[ct,"+","+","Domestic aviation"].sum()
co2.loc[ct,"international aviation"] = eurostat_co2[ct,"+","+","International aviation"].sum()
#doesn't include industrial process emissions or fuel processing/refining
co2.loc[ct,'industrial non-elec'] = eurostat_co2[ct,"+","Industry"].sum()
#doesn't include non-energy emissions
co2.loc[ct,'agriculture'] = eurostat_co2[ct,"+","+","Agriculture / Forestry"].sum()
return co2
def build_transport_data():
transport_data = pd.DataFrame(columns=["number cars","average fuel efficiency"],
index=population.index)
## collect number of cars
transport_data["number cars"] = idees["passenger cars"]
#CH from http://ec.europa.eu/eurostat/statistics-explained/index.php/Passenger_cars_in_the_EU#Luxembourg_has_the_highest_number_of_passenger_cars_per_inhabitant
transport_data.loc["CH","number cars"] = 4.136e6
missing = transport_data.index[transport_data["number cars"].isnull()]
print("Missing data on cars from:")
print(missing)
cars_pp = transport_data["number cars"]/population
transport_data.loc[missing,"number cars"] = cars_pp.mean()*population
## collect average fuel efficiency in kWh/km
transport_data["average fuel efficiency"] = idees["passenger car efficiency"]
missing = transport_data.index[transport_data["average fuel efficiency"].isnull()]
print("Missing data on fuel efficiency from:")
print(missing)
transport_data.loc[missing,"average fuel efficiency"] = transport_data["average fuel efficiency"].mean()
transport_data.to_csv(snakemake.output.transport_name)
return transport_data
if __name__ == "__main__":
# Detect running outside of snakemake and mock snakemake for testing
if 'snakemake' not in globals():
from vresutils import Dict
snakemake = Dict()
snakemake.output = Dict()
snakemake.output['energy_name'] = "data/energy_totals.csv"
snakemake.output['co2_name'] = "data/co2_totals.csv"
snakemake.output['transport_name'] = "data/transport_data.csv"
snakemake.input = Dict()
snakemake.input['nuts3_shapes'] = '../pypsa-eur/resources/nuts3_shapes.geojson'
nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index('index')
population = nuts3['pop'].groupby(nuts3.country).sum()
year = 2011
eurostat = build_eurostat(year)
swiss = build_swiss(year)
idees = build_idees(year)
build_energy_totals()
eea_co2 = build_eea_co2()
eurostat_co2 = build_eurostat_co2()
co2=build_co2_totals(eea_co2, eurostat_co2, year)
co2.to_csv(snakemake.output.co2_name)
build_transport_data()