From 22846d7b1b7d7bfb882ec08c4922d41fb904911d Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 9 Mar 2023 11:04:41 +0100 Subject: [PATCH] sector-coupled: run on subset of selected countries --- Snakefile | 4 +- scripts/add_existing_baseyear.py | 6 +- scripts/build_ammonia_production.py | 31 +---- scripts/build_energy_totals.py | 109 +++++++----------- scripts/build_industrial_distribution_key.py | 7 +- ...ustrial_energy_demand_per_country_today.py | 48 ++------ ...build_industrial_production_per_country.py | 44 ++----- scripts/build_population_layouts.py | 2 +- scripts/build_powerplants.py | 2 +- scripts/build_retro_cost.py | 2 +- scripts/plot_summary.py | 2 +- scripts/prepare_sector_network.py | 4 +- 12 files changed, 84 insertions(+), 177 deletions(-) diff --git a/Snakefile b/Snakefile index 09e92609..4661eb5a 100644 --- a/Snakefile +++ b/Snakefile @@ -69,8 +69,8 @@ rule purge: message: "Purging generated resources and results. Downloads are kept." run: - rmtree("resources/") - rmtree("results/") + rmtree("resources/", ignore_errors=True) + rmtree("results/", ignore_errors=True) rule dag: diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index ea2d5028..d119767e 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -431,10 +431,10 @@ def add_heating_capacities_installed_before_baseyear( ratio_residential = pd.Series( [ ( - n.loads_t.p_set.sum()["{} residential rural heat".format(node)] + n.loads_t.p_set.sum()[f"{node} residential rural heat"] / ( - n.loads_t.p_set.sum()["{} residential rural heat".format(node)] - + n.loads_t.p_set.sum()["{} services rural heat".format(node)] + n.loads_t.p_set.sum()[f"{node} residential rural heat"] + + n.loads_t.p_set.sum()[f"{node} services rural heat"] ) ) for node in nodal_df.index diff --git a/scripts/build_ammonia_production.py b/scripts/build_ammonia_production.py index 6e911235..3ba7175d 100644 --- a/scripts/build_ammonia_production.py +++ b/scripts/build_ammonia_production.py @@ -7,31 +7,10 @@ Build ammonia production. """ import pandas as pd +import country_converter as coco + +cc = coco.CountryConverter() -country_to_alpha2 = { - "Austriae": "AT", - "Bulgaria": "BG", - "Belgiume": "BE", - "Croatia": "HR", - "Czechia": "CZ", - "Estonia": "EE", - "Finland": "FI", - "France": "FR", - "Germany": "DE", - "Greece": "GR", - "Hungarye": "HU", - "Italye": "IT", - "Lithuania": "LT", - "Netherlands": "NL", - "Norwaye": "NO", - "Poland": "PL", - "Romania": "RO", - "Serbia": "RS", - "Slovakia": "SK", - "Spain": "ES", - "Switzerland": "CH", - "United Kingdom": "GB", -} if __name__ == "__main__": if "snakemake" not in globals(): @@ -48,10 +27,10 @@ if __name__ == "__main__": skipfooter=19, ) - ammonia.rename(country_to_alpha2, inplace=True) + ammonia.index = cc.convert(ammonia.index, to='iso2') years = [str(i) for i in range(2013, 2018)] - countries = country_to_alpha2.values() + countries = ammonia.index.intersection(snakemake.config["countries"]) ammonia = ammonia.loc[countries, years].astype(float) # convert from ktonN to ktonNH3 diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 7e3da9ff..814e7ed5 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -13,9 +13,12 @@ from functools import partial import geopandas as gpd import numpy as np import pandas as pd +import country_converter as coco from _helpers import mute_print from tqdm import tqdm +cc = coco.CountryConverter() + idx = pd.IndexSlice @@ -33,8 +36,7 @@ def reverse(dictionary): return {v: k for k, v in dictionary.items()} -# translations for Eurostat -eurostat_country_to_alpha2 = { +eurostat_codes = { "EU28": "EU", "EA19": "EA", "Belgium": "BE", @@ -81,38 +83,10 @@ eurostat_country_to_alpha2 = { "Switzerland": "CH", } -non_EU = ["NO", "CH", "ME", "MK", "RS", "BA", "AL"] idees_rename = {"GR": "EL", "GB": "UK"} -eu28 = [ - "FR", - "DE", - "GB", - "IT", - "ES", - "PL", - "SE", - "NL", - "BE", - "FI", - "CZ", - "DK", - "PT", - "RO", - "AT", - "BG", - "EE", - "GR", - "LV", - "HU", - "IE", - "SK", - "LT", - "HR", - "LU", - "SI", -] + ["CY", "MT"] +eu28 = cc.EU28as('ISO2').ISO2.tolist() eu28_eea = eu28.copy() eu28_eea.remove("GB") @@ -161,7 +135,7 @@ def build_eurostat(input_eurostat, countries, report_year, year): ) # sorted_index necessary for slicing - lookup = eurostat_country_to_alpha2 + lookup = eurostat_codes labelled_dfs = { lookup[df.columns[0]]: df for df in dfs.values() @@ -505,30 +479,32 @@ def build_energy_totals(countries, eurostat, swiss, idees): # http://www.ssb.no/en/energi-og-industri/statistikker/husenergi/hvert-3-aar/2014-07-14 # The main heating source for about 73 per cent of the households is based on electricity # => 26% is non-electric - elec_fraction = 0.73 - no_norway = df.drop("NO") + if "NO" in df: + elec_fraction = 0.73 - for sector in ["residential", "services"]: - # assume non-electric is heating - nonelectric = ( - df.loc["NO", f"total {sector}"] - df.loc["NO", f"electricity {sector}"] - ) - total_heating = nonelectric / (1 - elec_fraction) + no_norway = df.drop("NO") - for use in uses: - nonelectric_use = ( - no_norway[f"total {sector} {use}"] - - no_norway[f"electricity {sector} {use}"] - ) + for sector in ["residential", "services"]: + # assume non-electric is heating nonelectric = ( - no_norway[f"total {sector}"] - no_norway[f"electricity {sector}"] - ) - fraction = nonelectric_use.div(nonelectric).mean() - df.loc["NO", f"total {sector} {use}"] = total_heating * fraction - df.loc["NO", f"electricity {sector} {use}"] = ( - total_heating * fraction * elec_fraction + df.loc["NO", f"total {sector}"] - df.loc["NO", f"electricity {sector}"] ) + total_heating = nonelectric / (1 - elec_fraction) + + for use in uses: + nonelectric_use = ( + no_norway[f"total {sector} {use}"] + - no_norway[f"electricity {sector} {use}"] + ) + nonelectric = ( + no_norway[f"total {sector}"] - no_norway[f"electricity {sector}"] + ) + fraction = nonelectric_use.div(nonelectric).mean() + df.loc["NO", f"total {sector} {use}"] = total_heating * fraction + df.loc["NO", f"electricity {sector} {use}"] = ( + total_heating * fraction * elec_fraction + ) # Missing aviation @@ -687,7 +663,7 @@ def build_eurostat_co2(input_eurostat, countries, report_year, year=1990): def build_co2_totals(countries, eea_co2, eurostat_co2): co2 = eea_co2.reindex(countries) - for ct in countries.intersection(["BA", "RS", "AL", "ME", "MK"]): + for ct in pd.Index(countries).intersection(["BA", "RS", "AL", "ME", "MK"]): mappings = { "electricity": ( ct, @@ -724,27 +700,30 @@ def build_transport_data(countries, population, idees): transport_data["number cars"] = idees["passenger cars"] # CH from http://ec.europa.eu/eurostat/statistics-explained/index.php/Passenger_cars_in_the_EU#Luxembourg_has_the_highest_number_of_passenger_cars_per_inhabitant - transport_data.at["CH", "number cars"] = 4.136e6 + if "CH" in countries: + transport_data.at["CH", "number cars"] = 4.136e6 missing = transport_data.index[transport_data["number cars"].isna()] - logger.info( - f"Missing data on cars from:\n{list(missing)}\nFilling gaps with averaged data." - ) + if not missing.empty: + logger.info( + f"Missing data on cars from:\n{list(missing)}\nFilling gaps with averaged data." + ) - cars_pp = transport_data["number cars"] / population - transport_data.loc[missing, "number cars"] = cars_pp.mean() * population + cars_pp = transport_data["number cars"] / population + transport_data.loc[missing, "number cars"] = cars_pp.mean() * population # collect average fuel efficiency in kWh/km transport_data["average fuel efficiency"] = idees["passenger car efficiency"] missing = transport_data.index[transport_data["average fuel efficiency"].isna()] - logger.info( - f"Missing data on fuel efficiency from:\n{list(missing)}\nFilling gapswith averaged data." - ) + if not missing.empty: + logger.info( + f"Missing data on fuel efficiency from:\n{list(missing)}\nFilling gapswith averaged data." + ) - fill_values = transport_data["average fuel efficiency"].mean() - transport_data.loc[missing, "average fuel efficiency"] = fill_values + fill_values = transport_data["average fuel efficiency"].mean() + transport_data.loc[missing, "average fuel efficiency"] = fill_values return transport_data @@ -762,8 +741,8 @@ if __name__ == "__main__": nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index("index") population = nuts3["pop"].groupby(nuts3.country).sum() - countries = population.index - idees_countries = countries.intersection(eu28) + countries = snakemake.config["countries"] + idees_countries = pd.Index(countries).intersection(eu28) data_year = config["energy_totals_year"] report_year = snakemake.config["energy"]["eurostat_report_year"] diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index db2628b2..ad9e1489 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -99,13 +99,12 @@ def prepare_hotmaps_database(regions): return gdf -def build_nodal_distribution_key(hotmaps, regions): +def build_nodal_distribution_key(hotmaps, regions, countries): """ Build nodal distribution keys for each sector. """ sectors = hotmaps.Subsector.unique() - countries = regions.index.str[:2].unique() keys = pd.DataFrame(index=regions.index, columns=sectors, dtype=float) @@ -148,10 +147,12 @@ if __name__ == "__main__": logging.basicConfig(level=snakemake.config["logging"]["level"]) + countries = snakemake.config["countries"] + regions = gpd.read_file(snakemake.input.regions_onshore).set_index("name") hotmaps = prepare_hotmaps_database(regions) - keys = build_nodal_distribution_key(hotmaps, regions) + keys = build_nodal_distribution_key(hotmaps, regions, countries) keys.to_csv(snakemake.output.industrial_distribution_key) diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index fd99d9d6..1d64ecaf 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -11,8 +11,11 @@ import multiprocessing as mp from functools import partial import pandas as pd +import country_converter as coco from tqdm import tqdm +cc = coco.CountryConverter() + ktoe_to_twh = 0.011630 # name in JRC-IDEES Energy Balances @@ -56,36 +59,7 @@ fuels = { "Electricity": "electricity", } -eu28 = [ - "FR", - "DE", - "GB", - "IT", - "ES", - "PL", - "SE", - "NL", - "BE", - "FI", - "DK", - "PT", - "RO", - "AT", - "BG", - "EE", - "GR", - "LV", - "CZ", - "HU", - "IE", - "SK", - "LT", - "HR", - "LU", - "SI", - "CY", - "MT", -] +eu28 = cc.EU28as('ISO2').ISO2.tolist() jrc_names = {"GR": "EL", "GB": "UK"} @@ -154,7 +128,10 @@ def add_ammonia_energy_demand(demand): return demand -def add_non_eu28_industrial_energy_demand(demand): +def add_non_eu28_industrial_energy_demand(countries, demand): + non_eu28 = countries.difference(eu28) + if non_eu28.empty: + return demand # output in MtMaterial/a fn = snakemake.input.industrial_production_per_country production = pd.read_csv(fn, index_col=0) / 1e3 @@ -164,12 +141,10 @@ def add_non_eu28_industrial_energy_demand(demand): production["Basic chemicals (without ammonia)"] = production[chemicals].sum(axis=1) production.drop(columns=chemicals, inplace=True) - eu28_production = production.loc[eu28].sum() + eu28_production = production.loc[countries.intersection(eu28)].sum() eu28_energy = demand.groupby(level=1).sum() eu28_averages = eu28_energy / eu28_production - non_eu28 = production.index.symmetric_difference(eu28) - demand_non_eu28 = pd.concat( {k: v * eu28_averages for k, v in production.loc[non_eu28].iterrows()} ) @@ -206,12 +181,13 @@ if __name__ == "__main__": config = snakemake.config["industry"] year = config.get("reference_year", 2015) + countries = pd.Index(snakemake.config["countries"]) - demand = industrial_energy_demand(eu28, year) + demand = industrial_energy_demand(countries.intersection(eu28), year) demand = add_ammonia_energy_demand(demand) - demand = add_non_eu28_industrial_energy_demand(demand) + demand = add_non_eu28_industrial_energy_demand(countries, demand) # for format compatibility demand = demand.stack(dropna=False).unstack(level=[0, 2]) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 952c7382..97a116f5 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -16,9 +16,12 @@ import multiprocessing as mp import numpy as np import pandas as pd +import country_converter as coco from _helpers import mute_print from tqdm import tqdm +cc = coco.CountryConverter() + tj_to_ktoe = 0.0238845 ktoe_to_twh = 0.01163 @@ -36,41 +39,10 @@ sub_sheet_name_dict = { "Other Industrial Sectors": "OIS", } -non_EU = ["NO", "CH", "ME", "MK", "RS", "BA", "AL"] +eu28 = cc.EU28as('ISO2').ISO2.values jrc_names = {"GR": "EL", "GB": "UK"} -eu28 = [ - "FR", - "DE", - "GB", - "IT", - "ES", - "PL", - "SE", - "NL", - "BE", - "FI", - "DK", - "PT", - "RO", - "AT", - "BG", - "EE", - "GR", - "LV", - "CZ", - "HU", - "IE", - "SK", - "LT", - "HR", - "LU", - "SI", - "CY", - "MT", -] - sect2sub = { "Iron and steel": ["Electric arc", "Integrated steelworks"], "Chemicals Industry": [ @@ -233,10 +205,10 @@ def industry_production_per_country(country, year, eurostat_dir, jrc_dir): return df - ct = "EU28" if country in non_EU else country - demand = pd.concat([get_sector_data(s, ct) for s in sect2sub.keys()]) + ct = "EU28" if country not in eu28 else country + demand = pd.concat([get_sector_data(s, ct) for s in sect2sub]) - if country in non_EU: + if country not in eu28: demand *= get_energy_ratio(country, eurostat_dir, jrc_dir, year) demand.name = country @@ -309,7 +281,7 @@ if __name__ == "__main__": logging.basicConfig(level=snakemake.config["logging"]["level"]) - countries = non_EU + eu28 + countries = snakemake.config["countries"] year = snakemake.config["industry"]["reference_year"] diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index e3ac8f45..aa520eba 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -41,7 +41,7 @@ if __name__ == "__main__": # but imprecisions mean not perfect Iinv = cutout.indicatormatrix(nuts3.geometry) - countries = np.sort(nuts3.country.unique()) + countries = snakemake.config["countries"] urban_fraction = ( pd.read_csv( diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index 4158349e..f15a9082 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -115,7 +115,7 @@ if __name__ == "__main__": configure_logging(snakemake) n = pypsa.Network(snakemake.input.base_network) - countries = n.buses.country.unique() + countries = snakemake.config["countries"] ppl = ( pm.powerplants(from_url=True) diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index de0ccc16..67ccd635 100644 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -308,7 +308,7 @@ def prepare_building_stock_data(): u_values.set_index(["country_code", "subsector", "bage", "type"], inplace=True) # only take in config.yaml specified countries into account - countries = ct_total.index + countries = snakemake.config["countries"] area_tot = area_tot.loc[countries] return u_values, country_iso_dic, countries, area_tot, area diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index d7293715..01e3040e 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -456,7 +456,7 @@ def plot_carbon_budget_distribution(input_eurostat): ax1.set_xlim([1990, snakemake.config["scenario"]["planning_horizons"][-1] + 1]) path_cb = "results/" + snakemake.params.RDIR + "/csvs/" - countries = snakemake.confing["countries"] + countries = snakemake.config["countries"] e_1990 = co2_emissions_year(countries, input_eurostat, opts, year=1990) CO2_CAP = pd.read_csv(path_cb + "carbon_budget_distribution.csv", index_col=0) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index af94292b..9190e8d5 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -238,7 +238,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): carbon_budget = float(o[o.find("cb") + 2 : o.find("ex")]) r = float(o[o.find("ex") + 2 :]) - countries = n.buses.country.dropna().unique() + countries = snakemake.config["countries"] e_1990 = co2_emissions_year( countries, input_eurostat, opts, emissions_scope, report_year, year=1990 @@ -674,7 +674,7 @@ def add_dac(n, costs): def add_co2limit(n, Nyears=1.0, limit=0.0): logger.info(f"Adding CO2 budget limit as per unit of 1990 levels of {limit}") - countries = n.buses.country.dropna().unique() + countries = snakemake.config["countries"] sectors = emission_sectors_from_opts(opts)