Merge pull request #623 from PyPSA/sec-subset-countries

Run also sector-coupled build part for subset of countries

.github/workflows/ci.yaml

@@ -26,6 +26,7 @@ jobs:
   build:
 
     strategy:
+      fail-fast: false
       matrix:
         include:
           # Matrix required to handle caching with Mambaforge

Snakefile

@@ -69,8 +69,8 @@ rule purge:
     message:
         "Purging generated resources and results. Downloads are kept."
     run:
-        rmtree("resources/")
+        rmtree("resources/", ignore_errors=True)
-        rmtree("results/")
+        rmtree("results/", ignore_errors=True)
 
 
 rule dag:

scripts/add_existing_baseyear.py

@@ -431,10 +431,10 @@ def add_heating_capacities_installed_before_baseyear(
     ratio_residential = pd.Series(
         [
             (
-                n.loads_t.p_set.sum()["{} residential rural heat".format(node)]
+                n.loads_t.p_set.sum()[f"{node} residential rural heat"]
                 / (
-                    n.loads_t.p_set.sum()["{} residential rural heat".format(node)]
+                    n.loads_t.p_set.sum()[f"{node} residential rural heat"]
-                    + n.loads_t.p_set.sum()["{} services rural heat".format(node)]
+                    + n.loads_t.p_set.sum()[f"{node} services rural heat"]
                 )
             )
             for node in nodal_df.index

scripts/build_ammonia_production.py

@@ -6,32 +6,11 @@
 Build ammonia production.
 """
 
+import country_converter as coco
 import pandas as pd
 
-country_to_alpha2 = {
+cc = coco.CountryConverter()
-    "Austriae": "AT",
+
-    "Bulgaria": "BG",
-    "Belgiume": "BE",
-    "Croatia": "HR",
-    "Czechia": "CZ",
-    "Estonia": "EE",
-    "Finland": "FI",
-    "France": "FR",
-    "Germany": "DE",
-    "Greece": "GR",
-    "Hungarye": "HU",
-    "Italye": "IT",
-    "Lithuania": "LT",
-    "Netherlands": "NL",
-    "Norwaye": "NO",
-    "Poland": "PL",
-    "Romania": "RO",
-    "Serbia": "RS",
-    "Slovakia": "SK",
-    "Spain": "ES",
-    "Switzerland": "CH",
-    "United Kingdom": "GB",
-}
 
 if __name__ == "__main__":
     if "snakemake" not in globals():
@@ -48,10 +27,10 @@ if __name__ == "__main__":
         skipfooter=19,
     )
 
-    ammonia.rename(country_to_alpha2, inplace=True)
+    ammonia.index = cc.convert(ammonia.index, to="iso2")
 
     years = [str(i) for i in range(2013, 2018)]
-    countries = country_to_alpha2.values()
+    countries = ammonia.index.intersection(snakemake.config["countries"])
     ammonia = ammonia.loc[countries, years].astype(float)
 
     # convert from ktonN to ktonNH3

scripts/build_energy_totals.py

@@ -10,12 +10,15 @@ logger = logging.getLogger(__name__)
 import multiprocessing as mp
 from functools import partial
 
+import country_converter as coco
 import geopandas as gpd
 import numpy as np
 import pandas as pd
 from _helpers import mute_print
 from tqdm import tqdm
 
+cc = coco.CountryConverter()
+
 idx = pd.IndexSlice
 
 
@@ -33,8 +36,7 @@ def reverse(dictionary):
     return {v: k for k, v in dictionary.items()}
 
 
-# translations for Eurostat
+eurostat_codes = {
-eurostat_country_to_alpha2 = {
     "EU28": "EU",
     "EA19": "EA",
     "Belgium": "BE",
@@ -81,38 +83,10 @@ eurostat_country_to_alpha2 = {
     "Switzerland": "CH",
 }
 
-non_EU = ["NO", "CH", "ME", "MK", "RS", "BA", "AL"]
 
 idees_rename = {"GR": "EL", "GB": "UK"}
 
-eu28 = [
+eu28 = cc.EU28as("ISO2").ISO2.tolist()
-    "FR",
-    "DE",
-    "GB",
-    "IT",
-    "ES",
-    "PL",
-    "SE",
-    "NL",
-    "BE",
-    "FI",
-    "CZ",
-    "DK",
-    "PT",
-    "RO",
-    "AT",
-    "BG",
-    "EE",
-    "GR",
-    "LV",
-    "HU",
-    "IE",
-    "SK",
-    "LT",
-    "HR",
-    "LU",
-    "SI",
-] + ["CY", "MT"]
 
 eu28_eea = eu28.copy()
 eu28_eea.remove("GB")
@@ -161,7 +135,7 @@ def build_eurostat(input_eurostat, countries, report_year, year):
         )
 
     # sorted_index necessary for slicing
-    lookup = eurostat_country_to_alpha2
+    lookup = eurostat_codes
     labelled_dfs = {
         lookup[df.columns[0]]: df
         for df in dfs.values()
@@ -505,6 +479,8 @@ def build_energy_totals(countries, eurostat, swiss, idees):
     # http://www.ssb.no/en/energi-og-industri/statistikker/husenergi/hvert-3-aar/2014-07-14
     # The main heating source for about 73 per cent of the households is based on electricity
     # => 26% is non-electric
+
+    if "NO" in df:
         elec_fraction = 0.73
 
         no_norway = df.drop("NO")
@@ -687,7 +663,7 @@ def build_eurostat_co2(input_eurostat, countries, report_year, year=1990):
 def build_co2_totals(countries, eea_co2, eurostat_co2):
     co2 = eea_co2.reindex(countries)
 
-    for ct in countries.intersection(["BA", "RS", "AL", "ME", "MK"]):
+    for ct in pd.Index(countries).intersection(["BA", "RS", "AL", "ME", "MK"]):
         mappings = {
             "electricity": (
                 ct,
@@ -724,9 +700,11 @@ def build_transport_data(countries, population, idees):
     transport_data["number cars"] = idees["passenger cars"]
 
     # CH from http://ec.europa.eu/eurostat/statistics-explained/index.php/Passenger_cars_in_the_EU#Luxembourg_has_the_highest_number_of_passenger_cars_per_inhabitant
+    if "CH" in countries:
         transport_data.at["CH", "number cars"] = 4.136e6
 
     missing = transport_data.index[transport_data["number cars"].isna()]
+    if not missing.empty:
         logger.info(
             f"Missing data on cars from:\n{list(missing)}\nFilling gaps with averaged data."
         )
@@ -739,6 +717,7 @@ def build_transport_data(countries, population, idees):
     transport_data["average fuel efficiency"] = idees["passenger car efficiency"]
 
     missing = transport_data.index[transport_data["average fuel efficiency"].isna()]
+    if not missing.empty:
         logger.info(
             f"Missing data on fuel efficiency from:\n{list(missing)}\nFilling gapswith averaged data."
         )
@@ -762,8 +741,8 @@ if __name__ == "__main__":
     nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index("index")
     population = nuts3["pop"].groupby(nuts3.country).sum()
 
-    countries = population.index
+    countries = snakemake.config["countries"]
-    idees_countries = countries.intersection(eu28)
+    idees_countries = pd.Index(countries).intersection(eu28)
 
     data_year = config["energy_totals_year"]
     report_year = snakemake.config["energy"]["eurostat_report_year"]

scripts/build_industrial_distribution_key.py

@@ -99,13 +99,12 @@ def prepare_hotmaps_database(regions):
     return gdf
 
 
-def build_nodal_distribution_key(hotmaps, regions):
+def build_nodal_distribution_key(hotmaps, regions, countries):
     """
     Build nodal distribution keys for each sector.
     """
 
     sectors = hotmaps.Subsector.unique()
-    countries = regions.index.str[:2].unique()
 
     keys = pd.DataFrame(index=regions.index, columns=sectors, dtype=float)
 
@@ -148,10 +147,12 @@ if __name__ == "__main__":
 
     logging.basicConfig(level=snakemake.config["logging"]["level"])
 
+    countries = snakemake.config["countries"]
+
     regions = gpd.read_file(snakemake.input.regions_onshore).set_index("name")
 
     hotmaps = prepare_hotmaps_database(regions)
 
-    keys = build_nodal_distribution_key(hotmaps, regions)
+    keys = build_nodal_distribution_key(hotmaps, regions, countries)
 
     keys.to_csv(snakemake.output.industrial_distribution_key)

scripts/build_industrial_energy_demand_per_country_today.py

@@ -10,9 +10,12 @@ Build industrial energy demand per country.
 import multiprocessing as mp
 from functools import partial
 
+import country_converter as coco
 import pandas as pd
 from tqdm import tqdm
 
+cc = coco.CountryConverter()
+
 ktoe_to_twh = 0.011630
 
 # name in JRC-IDEES Energy Balances
@@ -56,36 +59,7 @@ fuels = {
     "Electricity": "electricity",
 }
 
-eu28 = [
+eu28 = cc.EU28as("ISO2").ISO2.tolist()
-    "FR",
-    "DE",
-    "GB",
-    "IT",
-    "ES",
-    "PL",
-    "SE",
-    "NL",
-    "BE",
-    "FI",
-    "DK",
-    "PT",
-    "RO",
-    "AT",
-    "BG",
-    "EE",
-    "GR",
-    "LV",
-    "CZ",
-    "HU",
-    "IE",
-    "SK",
-    "LT",
-    "HR",
-    "LU",
-    "SI",
-    "CY",
-    "MT",
-]
 
 jrc_names = {"GR": "EL", "GB": "UK"}
 
@@ -154,7 +128,10 @@ def add_ammonia_energy_demand(demand):
     return demand
 
 
-def add_non_eu28_industrial_energy_demand(demand):
+def add_non_eu28_industrial_energy_demand(countries, demand):
+    non_eu28 = countries.difference(eu28)
+    if non_eu28.empty:
+        return demand
     # output in MtMaterial/a
     fn = snakemake.input.industrial_production_per_country
     production = pd.read_csv(fn, index_col=0) / 1e3
@@ -164,12 +141,10 @@ def add_non_eu28_industrial_energy_demand(demand):
     production["Basic chemicals (without ammonia)"] = production[chemicals].sum(axis=1)
     production.drop(columns=chemicals, inplace=True)
 
-    eu28_production = production.loc[eu28].sum()
+    eu28_production = production.loc[countries.intersection(eu28)].sum()
     eu28_energy = demand.groupby(level=1).sum()
     eu28_averages = eu28_energy / eu28_production
 
-    non_eu28 = production.index.symmetric_difference(eu28)
-
     demand_non_eu28 = pd.concat(
         {k: v * eu28_averages for k, v in production.loc[non_eu28].iterrows()}
     )
@@ -206,12 +181,13 @@ if __name__ == "__main__":
 
     config = snakemake.config["industry"]
     year = config.get("reference_year", 2015)
+    countries = pd.Index(snakemake.config["countries"])
 
-    demand = industrial_energy_demand(eu28, year)
+    demand = industrial_energy_demand(countries.intersection(eu28), year)
 
     demand = add_ammonia_energy_demand(demand)
 
-    demand = add_non_eu28_industrial_energy_demand(demand)
+    demand = add_non_eu28_industrial_energy_demand(countries, demand)
 
     # for format compatibility
     demand = demand.stack(dropna=False).unstack(level=[0, 2])

scripts/build_industrial_production_per_country.py

@@ -14,11 +14,14 @@ logger = logging.getLogger(__name__)
 
 import multiprocessing as mp
 
+import country_converter as coco
 import numpy as np
 import pandas as pd
 from _helpers import mute_print
 from tqdm import tqdm
 
+cc = coco.CountryConverter()
+
 tj_to_ktoe = 0.0238845
 ktoe_to_twh = 0.01163
 
@@ -36,41 +39,10 @@ sub_sheet_name_dict = {
     "Other Industrial Sectors": "OIS",
 }
 
-non_EU = ["NO", "CH", "ME", "MK", "RS", "BA", "AL"]
+eu28 = cc.EU28as("ISO2").ISO2.values
 
 jrc_names = {"GR": "EL", "GB": "UK"}
 
-eu28 = [
-    "FR",
-    "DE",
-    "GB",
-    "IT",
-    "ES",
-    "PL",
-    "SE",
-    "NL",
-    "BE",
-    "FI",
-    "DK",
-    "PT",
-    "RO",
-    "AT",
-    "BG",
-    "EE",
-    "GR",
-    "LV",
-    "CZ",
-    "HU",
-    "IE",
-    "SK",
-    "LT",
-    "HR",
-    "LU",
-    "SI",
-    "CY",
-    "MT",
-]
-
 sect2sub = {
     "Iron and steel": ["Electric arc", "Integrated steelworks"],
     "Chemicals Industry": [
@@ -233,10 +205,10 @@ def industry_production_per_country(country, year, eurostat_dir, jrc_dir):
 
         return df
 
-    ct = "EU28" if country in non_EU else country
+    ct = "EU28" if country not in eu28 else country
-    demand = pd.concat([get_sector_data(s, ct) for s in sect2sub.keys()])
+    demand = pd.concat([get_sector_data(s, ct) for s in sect2sub])
 
-    if country in non_EU:
+    if country not in eu28:
         demand *= get_energy_ratio(country, eurostat_dir, jrc_dir, year)
 
     demand.name = country
@@ -309,7 +281,7 @@ if __name__ == "__main__":
 
     logging.basicConfig(level=snakemake.config["logging"]["level"])
 
-    countries = non_EU + eu28
+    countries = snakemake.config["countries"]
 
     year = snakemake.config["industry"]["reference_year"]
 

scripts/build_powerplants.py

@@ -115,7 +115,7 @@ if __name__ == "__main__":
     configure_logging(snakemake)
 
     n = pypsa.Network(snakemake.input.base_network)
-    countries = n.buses.country.unique()
+    countries = snakemake.config["countries"]
 
     ppl = (
         pm.powerplants(from_url=True)

scripts/build_retro_cost.py

@@ -308,7 +308,7 @@ def prepare_building_stock_data():
     u_values.set_index(["country_code", "subsector", "bage", "type"], inplace=True)
 
     #  only take in config.yaml specified countries into account
-    countries = ct_total.index
+    countries = snakemake.config["countries"]
     area_tot = area_tot.loc[countries]
 
     return u_values, country_iso_dic, countries, area_tot, area

scripts/plot_summary.py

@@ -456,7 +456,7 @@ def plot_carbon_budget_distribution(input_eurostat):
     ax1.set_xlim([1990, snakemake.config["scenario"]["planning_horizons"][-1] + 1])
 
     path_cb = "results/" + snakemake.params.RDIR + "/csvs/"
-    countries = snakemake.confing["countries"]
+    countries = snakemake.config["countries"]
     e_1990 = co2_emissions_year(countries, input_eurostat, opts, year=1990)
     CO2_CAP = pd.read_csv(path_cb + "carbon_budget_distribution.csv", index_col=0)
 

scripts/prepare_sector_network.py

@@ -238,7 +238,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year):
         carbon_budget = float(o[o.find("cb") + 2 : o.find("ex")])
         r = float(o[o.find("ex") + 2 :])
 
-    countries = n.buses.country.dropna().unique()
+    countries = snakemake.config["countries"]
 
     e_1990 = co2_emissions_year(
         countries, input_eurostat, opts, emissions_scope, report_year, year=1990
@@ -674,7 +674,7 @@ def add_dac(n, costs):
 def add_co2limit(n, Nyears=1.0, limit=0.0):
     logger.info(f"Adding CO2 budget limit as per unit of 1990 levels of {limit}")
 
-    countries = n.buses.country.dropna().unique()
+    countries = snakemake.config["countries"]
 
     sectors = emission_sectors_from_opts(opts)
 

test/config.myopic.yaml

@@ -15,9 +15,9 @@ scenario:
   ll:
   - v1.5
   clusters:
-  - 5
+  - 4
   sector_opts:
-  - 191H-T-H-B-I-A-solar+p3-dist1
+  - 40H-T-H-B-I-A-solar+p3-dist1
   planning_horizons:
   - 2030
   - 2040
@@ -27,7 +27,7 @@ countries: ['BE']
 
 snapshots:
   start: "2013-03-01"
-  end: "2013-04-01"
+  end: "2013-03-08"
 
 electricity:
   renewable_carriers: [solar, onwind, offwind-ac, offwind-dc]

test/config.overnight.yaml

@@ -14,9 +14,9 @@ scenario:
   ll:
   - v1.5
   clusters:
-  - 5
+  - 4
   sector_opts:
-  - CO2L0-191H-T-H-B-I-A-solar+p3-dist1
+  - CO2L0-40H-T-H-B-I-A-solar+p3-dist1
   planning_horizons:
   - 2030
 
@@ -24,7 +24,7 @@ countries: ['BE']
 
 snapshots:
   start: "2013-03-01"
-  end: "2013-04-01"
+  end: "2013-03-08"
 
 electricity:
   renewable_carriers: [solar, onwind, offwind-ac, offwind-dc]

test/config.test1.yaml

@@ -13,9 +13,9 @@ run:
 
 scenario:
   clusters:
-  - 5
+  - 4
   opts:
-  - Co2L-24H
+  - Co2L-40H
 
 countries: ['BE']