sector-coupled: run on subset of selected countries

2023-03-09 11:04:41 +01:00 · 2023-03-09 11:04:41 +01:00 · 22846d7b1b
commit 22846d7b1b
parent 483f3e1b82
12 changed files with 84 additions and 177 deletions
--- a/4
+++ b/4
@ -69,8 +69,8 @@ rule purge:
    message:
        "Purging generated resources and results. Downloads are kept."
    run:
-        rmtree("resources/")
-        rmtree("results/")
+        rmtree("resources/", ignore_errors=True)
+        rmtree("results/", ignore_errors=True)


 rule dag:
--- a/scripts/add_existing_baseyear.py
+++ b/scripts/add_existing_baseyear.py
@ -431,10 +431,10 @@ def add_heating_capacities_installed_before_baseyear(
    ratio_residential = pd.Series(
        [
            (
-                n.loads_t.p_set.sum()["{} residential rural heat".format(node)]
+                n.loads_t.p_set.sum()[f"{node} residential rural heat"]
                / (
-                    n.loads_t.p_set.sum()["{} residential rural heat".format(node)]
-                    + n.loads_t.p_set.sum()["{} services rural heat".format(node)]
+                    n.loads_t.p_set.sum()[f"{node} residential rural heat"]
+                    + n.loads_t.p_set.sum()[f"{node} services rural heat"]
                )
            )
            for node in nodal_df.index
--- a/scripts/build_ammonia_production.py
+++ b/scripts/build_ammonia_production.py
@ -7,31 +7,10 @@ Build ammonia production.
 """

 import pandas as pd
+import country_converter as coco
+
+cc = coco.CountryConverter()

-country_to_alpha2 = {
-    "Austriae": "AT",
-    "Bulgaria": "BG",
-    "Belgiume": "BE",
-    "Croatia": "HR",
-    "Czechia": "CZ",
-    "Estonia": "EE",
-    "Finland": "FI",
-    "France": "FR",
-    "Germany": "DE",
-    "Greece": "GR",
-    "Hungarye": "HU",
-    "Italye": "IT",
-    "Lithuania": "LT",
-    "Netherlands": "NL",
-    "Norwaye": "NO",
-    "Poland": "PL",
-    "Romania": "RO",
-    "Serbia": "RS",
-    "Slovakia": "SK",
-    "Spain": "ES",
-    "Switzerland": "CH",
-    "United Kingdom": "GB",
-}

 if __name__ == "__main__":
    if "snakemake" not in globals():
@ -48,10 +27,10 @@ if __name__ == "__main__":
        skipfooter=19,
    )

-    ammonia.rename(country_to_alpha2, inplace=True)
+    ammonia.index = cc.convert(ammonia.index, to='iso2')

    years = [str(i) for i in range(2013, 2018)]
-    countries = country_to_alpha2.values()
+    countries = ammonia.index.intersection(snakemake.config["countries"])
    ammonia = ammonia.loc[countries, years].astype(float)

    # convert from ktonN to ktonNH3
--- a/scripts/build_energy_totals.py
+++ b/scripts/build_energy_totals.py
@ -13,9 +13,12 @@ from functools import partial
 import geopandas as gpd
 import numpy as np
 import pandas as pd
+import country_converter as coco
 from _helpers import mute_print
 from tqdm import tqdm

+cc = coco.CountryConverter()
+
 idx = pd.IndexSlice


@ -33,8 +36,7 @@ def reverse(dictionary):
    return {v: k for k, v in dictionary.items()}


-# translations for Eurostat
-eurostat_country_to_alpha2 = {
+eurostat_codes = {
    "EU28": "EU",
    "EA19": "EA",
    "Belgium": "BE",
@ -81,38 +83,10 @@ eurostat_country_to_alpha2 = {
    "Switzerland": "CH",
 }

-non_EU = ["NO", "CH", "ME", "MK", "RS", "BA", "AL"]

 idees_rename = {"GR": "EL", "GB": "UK"}

-eu28 = [
-    "FR",
-    "DE",
-    "GB",
-    "IT",
-    "ES",
-    "PL",
-    "SE",
-    "NL",
-    "BE",
-    "FI",
-    "CZ",
-    "DK",
-    "PT",
-    "RO",
-    "AT",
-    "BG",
-    "EE",
-    "GR",
-    "LV",
-    "HU",
-    "IE",
-    "SK",
-    "LT",
-    "HR",
-    "LU",
-    "SI",
-] + ["CY", "MT"]
+eu28 = cc.EU28as('ISO2').ISO2.tolist()

 eu28_eea = eu28.copy()
 eu28_eea.remove("GB")
@ -161,7 +135,7 @@ def build_eurostat(input_eurostat, countries, report_year, year):
        )

    # sorted_index necessary for slicing
-    lookup = eurostat_country_to_alpha2
+    lookup = eurostat_codes
    labelled_dfs = {
        lookup[df.columns[0]]: df
        for df in dfs.values()
@ -505,30 +479,32 @@ def build_energy_totals(countries, eurostat, swiss, idees):
    # http://www.ssb.no/en/energi-og-industri/statistikker/husenergi/hvert-3-aar/2014-07-14
    # The main heating source for about 73 per cent of the households is based on electricity
    # => 26% is non-electric
-    elec_fraction = 0.73

-    no_norway = df.drop("NO")
+    if "NO" in df:
+        elec_fraction = 0.73

-    for sector in ["residential", "services"]:
-        # assume non-electric is heating
-        nonelectric = (
-            df.loc["NO", f"total {sector}"] - df.loc["NO", f"electricity {sector}"]
-        )
-        total_heating = nonelectric / (1 - elec_fraction)
+        no_norway = df.drop("NO")

-        for use in uses:
-            nonelectric_use = (
-                no_norway[f"total {sector} {use}"]
-                - no_norway[f"electricity {sector} {use}"]
-            )
+        for sector in ["residential", "services"]:
+            # assume non-electric is heating
            nonelectric = (
-                no_norway[f"total {sector}"] - no_norway[f"electricity {sector}"]
-            )
-            fraction = nonelectric_use.div(nonelectric).mean()
-            df.loc["NO", f"total {sector} {use}"] = total_heating * fraction
-            df.loc["NO", f"electricity {sector} {use}"] = (
-                total_heating * fraction * elec_fraction
+                df.loc["NO", f"total {sector}"] - df.loc["NO", f"electricity {sector}"]
            )
+            total_heating = nonelectric / (1 - elec_fraction)
+
+            for use in uses:
+                nonelectric_use = (
+                    no_norway[f"total {sector} {use}"]
+                    - no_norway[f"electricity {sector} {use}"]
+                )
+                nonelectric = (
+                    no_norway[f"total {sector}"] - no_norway[f"electricity {sector}"]
+                )
+                fraction = nonelectric_use.div(nonelectric).mean()
+                df.loc["NO", f"total {sector} {use}"] = total_heating * fraction
+                df.loc["NO", f"electricity {sector} {use}"] = (
+                    total_heating * fraction * elec_fraction
+                )

    # Missing aviation

@ -687,7 +663,7 @@ def build_eurostat_co2(input_eurostat, countries, report_year, year=1990):
 def build_co2_totals(countries, eea_co2, eurostat_co2):
    co2 = eea_co2.reindex(countries)

-    for ct in countries.intersection(["BA", "RS", "AL", "ME", "MK"]):
+    for ct in pd.Index(countries).intersection(["BA", "RS", "AL", "ME", "MK"]):
        mappings = {
            "electricity": (
                ct,
@ -724,27 +700,30 @@ def build_transport_data(countries, population, idees):
    transport_data["number cars"] = idees["passenger cars"]

    # CH from http://ec.europa.eu/eurostat/statistics-explained/index.php/Passenger_cars_in_the_EU#Luxembourg_has_the_highest_number_of_passenger_cars_per_inhabitant
-    transport_data.at["CH", "number cars"] = 4.136e6
+    if "CH" in countries:
+        transport_data.at["CH", "number cars"] = 4.136e6

    missing = transport_data.index[transport_data["number cars"].isna()]
-    logger.info(
-        f"Missing data on cars from:\n{list(missing)}\nFilling gaps with averaged data."
-    )
+    if not missing.empty:
+        logger.info(
+            f"Missing data on cars from:\n{list(missing)}\nFilling gaps with averaged data."
+        )

-    cars_pp = transport_data["number cars"] / population
-    transport_data.loc[missing, "number cars"] = cars_pp.mean() * population
+        cars_pp = transport_data["number cars"] / population
+        transport_data.loc[missing, "number cars"] = cars_pp.mean() * population

    # collect average fuel efficiency in kWh/km

    transport_data["average fuel efficiency"] = idees["passenger car efficiency"]

    missing = transport_data.index[transport_data["average fuel efficiency"].isna()]
-    logger.info(
-        f"Missing data on fuel efficiency from:\n{list(missing)}\nFilling gapswith averaged data."
-    )
+    if not missing.empty:
+        logger.info(
+            f"Missing data on fuel efficiency from:\n{list(missing)}\nFilling gapswith averaged data."
+        )

-    fill_values = transport_data["average fuel efficiency"].mean()
-    transport_data.loc[missing, "average fuel efficiency"] = fill_values
+        fill_values = transport_data["average fuel efficiency"].mean()
+        transport_data.loc[missing, "average fuel efficiency"] = fill_values

    return transport_data

@ -762,8 +741,8 @@ if __name__ == "__main__":
    nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index("index")
    population = nuts3["pop"].groupby(nuts3.country).sum()

-    countries = population.index
-    idees_countries = countries.intersection(eu28)
+    countries = snakemake.config["countries"]
+    idees_countries = pd.Index(countries).intersection(eu28)

    data_year = config["energy_totals_year"]
    report_year = snakemake.config["energy"]["eurostat_report_year"]
--- a/scripts/build_industrial_distribution_key.py
+++ b/scripts/build_industrial_distribution_key.py
@ -99,13 +99,12 @@ def prepare_hotmaps_database(regions):
    return gdf


-def build_nodal_distribution_key(hotmaps, regions):
+def build_nodal_distribution_key(hotmaps, regions, countries):
    """
    Build nodal distribution keys for each sector.
    """

    sectors = hotmaps.Subsector.unique()
-    countries = regions.index.str[:2].unique()

    keys = pd.DataFrame(index=regions.index, columns=sectors, dtype=float)

@ -148,10 +147,12 @@ if __name__ == "__main__":

    logging.basicConfig(level=snakemake.config["logging"]["level"])

+    countries = snakemake.config["countries"]
+
    regions = gpd.read_file(snakemake.input.regions_onshore).set_index("name")

    hotmaps = prepare_hotmaps_database(regions)

-    keys = build_nodal_distribution_key(hotmaps, regions)
+    keys = build_nodal_distribution_key(hotmaps, regions, countries)

    keys.to_csv(snakemake.output.industrial_distribution_key)
--- a/scripts/build_industrial_energy_demand_per_country_today.py
+++ b/scripts/build_industrial_energy_demand_per_country_today.py
@ -11,8 +11,11 @@ import multiprocessing as mp
 from functools import partial

 import pandas as pd
+import country_converter as coco
 from tqdm import tqdm

+cc = coco.CountryConverter()
+
 ktoe_to_twh = 0.011630

 # name in JRC-IDEES Energy Balances
@ -56,36 +59,7 @@ fuels = {
    "Electricity": "electricity",
 }

-eu28 = [
-    "FR",
-    "DE",
-    "GB",
-    "IT",
-    "ES",
-    "PL",
-    "SE",
-    "NL",
-    "BE",
-    "FI",
-    "DK",
-    "PT",
-    "RO",
-    "AT",
-    "BG",
-    "EE",
-    "GR",
-    "LV",
-    "CZ",
-    "HU",
-    "IE",
-    "SK",
-    "LT",
-    "HR",
-    "LU",
-    "SI",
-    "CY",
-    "MT",
-]
+eu28 = cc.EU28as('ISO2').ISO2.tolist()

 jrc_names = {"GR": "EL", "GB": "UK"}

@ -154,7 +128,10 @@ def add_ammonia_energy_demand(demand):
    return demand


-def add_non_eu28_industrial_energy_demand(demand):
+def add_non_eu28_industrial_energy_demand(countries, demand):
+    non_eu28 = countries.difference(eu28)
+    if non_eu28.empty:
+        return demand
    # output in MtMaterial/a
    fn = snakemake.input.industrial_production_per_country
    production = pd.read_csv(fn, index_col=0) / 1e3
@ -164,12 +141,10 @@ def add_non_eu28_industrial_energy_demand(demand):
    production["Basic chemicals (without ammonia)"] = production[chemicals].sum(axis=1)
    production.drop(columns=chemicals, inplace=True)

-    eu28_production = production.loc[eu28].sum()
+    eu28_production = production.loc[countries.intersection(eu28)].sum()
    eu28_energy = demand.groupby(level=1).sum()
    eu28_averages = eu28_energy / eu28_production

-    non_eu28 = production.index.symmetric_difference(eu28)
-
    demand_non_eu28 = pd.concat(
        {k: v * eu28_averages for k, v in production.loc[non_eu28].iterrows()}
    )
@ -206,12 +181,13 @@ if __name__ == "__main__":

    config = snakemake.config["industry"]
    year = config.get("reference_year", 2015)
+    countries = pd.Index(snakemake.config["countries"])

-    demand = industrial_energy_demand(eu28, year)
+    demand = industrial_energy_demand(countries.intersection(eu28), year)

    demand = add_ammonia_energy_demand(demand)

-    demand = add_non_eu28_industrial_energy_demand(demand)
+    demand = add_non_eu28_industrial_energy_demand(countries, demand)

    # for format compatibility
    demand = demand.stack(dropna=False).unstack(level=[0, 2])
--- a/scripts/build_industrial_production_per_country.py
+++ b/scripts/build_industrial_production_per_country.py
@ -16,9 +16,12 @@ import multiprocessing as mp

 import numpy as np
 import pandas as pd
+import country_converter as coco
 from _helpers import mute_print
 from tqdm import tqdm

+cc = coco.CountryConverter()
+
 tj_to_ktoe = 0.0238845
 ktoe_to_twh = 0.01163

@ -36,41 +39,10 @@ sub_sheet_name_dict = {
    "Other Industrial Sectors": "OIS",
 }

-non_EU = ["NO", "CH", "ME", "MK", "RS", "BA", "AL"]
+eu28 = cc.EU28as('ISO2').ISO2.values

 jrc_names = {"GR": "EL", "GB": "UK"}

-eu28 = [
-    "FR",
-    "DE",
-    "GB",
-    "IT",
-    "ES",
-    "PL",
-    "SE",
-    "NL",
-    "BE",
-    "FI",
-    "DK",
-    "PT",
-    "RO",
-    "AT",
-    "BG",
-    "EE",
-    "GR",
-    "LV",
-    "CZ",
-    "HU",
-    "IE",
-    "SK",
-    "LT",
-    "HR",
-    "LU",
-    "SI",
-    "CY",
-    "MT",
-]
-
 sect2sub = {
    "Iron and steel": ["Electric arc", "Integrated steelworks"],
    "Chemicals Industry": [
@ -233,10 +205,10 @@ def industry_production_per_country(country, year, eurostat_dir, jrc_dir):

        return df

-    ct = "EU28" if country in non_EU else country
-    demand = pd.concat([get_sector_data(s, ct) for s in sect2sub.keys()])
+    ct = "EU28" if country not in eu28 else country
+    demand = pd.concat([get_sector_data(s, ct) for s in sect2sub])

-    if country in non_EU:
+    if country not in eu28:
        demand *= get_energy_ratio(country, eurostat_dir, jrc_dir, year)

    demand.name = country
@ -309,7 +281,7 @@ if __name__ == "__main__":

    logging.basicConfig(level=snakemake.config["logging"]["level"])

-    countries = non_EU + eu28
+    countries = snakemake.config["countries"]

    year = snakemake.config["industry"]["reference_year"]

--- a/scripts/build_population_layouts.py
+++ b/scripts/build_population_layouts.py
@ -41,7 +41,7 @@ if __name__ == "__main__":
    # but imprecisions mean not perfect
    Iinv = cutout.indicatormatrix(nuts3.geometry)

-    countries = np.sort(nuts3.country.unique())
+    countries = snakemake.config["countries"]

    urban_fraction = (
        pd.read_csv(
--- a/scripts/build_powerplants.py
+++ b/scripts/build_powerplants.py
@ -115,7 +115,7 @@ if __name__ == "__main__":
    configure_logging(snakemake)

    n = pypsa.Network(snakemake.input.base_network)
-    countries = n.buses.country.unique()
+    countries = snakemake.config["countries"]

    ppl = (
        pm.powerplants(from_url=True)
--- a/scripts/build_retro_cost.py
+++ b/scripts/build_retro_cost.py
@ -308,7 +308,7 @@ def prepare_building_stock_data():
    u_values.set_index(["country_code", "subsector", "bage", "type"], inplace=True)

    #  only take in config.yaml specified countries into account
-    countries = ct_total.index
+    countries = snakemake.config["countries"]
    area_tot = area_tot.loc[countries]

    return u_values, country_iso_dic, countries, area_tot, area
--- a/scripts/plot_summary.py
+++ b/scripts/plot_summary.py
@ -456,7 +456,7 @@ def plot_carbon_budget_distribution(input_eurostat):
    ax1.set_xlim([1990, snakemake.config["scenario"]["planning_horizons"][-1] + 1])

    path_cb = "results/" + snakemake.params.RDIR + "/csvs/"
-    countries = snakemake.confing["countries"]
+    countries = snakemake.config["countries"]
    e_1990 = co2_emissions_year(countries, input_eurostat, opts, year=1990)
    CO2_CAP = pd.read_csv(path_cb + "carbon_budget_distribution.csv", index_col=0)

--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@ -238,7 +238,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year):
        carbon_budget = float(o[o.find("cb") + 2 : o.find("ex")])
        r = float(o[o.find("ex") + 2 :])

-    countries = n.buses.country.dropna().unique()
+    countries = snakemake.config["countries"]

    e_1990 = co2_emissions_year(
        countries, input_eurostat, opts, emissions_scope, report_year, year=1990
@ -674,7 +674,7 @@ def add_dac(n, costs):
 def add_co2limit(n, Nyears=1.0, limit=0.0):
    logger.info(f"Adding CO2 budget limit as per unit of 1990 levels of {limit}")

-    countries = n.buses.country.dropna().unique()
+    countries = snakemake.config["countries"]

    sectors = emission_sectors_from_opts(opts)