Merge pull request #653 from PyPSA/complete-carrier-set

Complete carrier set

config/config.default.yaml

@@ -790,6 +790,11 @@ plotting:
     solar: "#f9d002"
     solar PV: "#f9d002"
     solar thermal: '#ffbf2b'
+    residential rural solar thermal: '#f1c069'
+    services rural solar thermal: '#eabf61'
+    residential urban decentral solar thermal: '#e5bc5a'
+    services urban decentral solar thermal: '#dfb953'
+    urban central solar thermal: '#d7b24c'
     solar rooftop: '#ffea80'
     # gas
     OCGT: '#e0986c'
@@ -798,9 +803,15 @@ plotting:
     gas boiler: '#db6a25'
     gas boilers: '#db6a25'
     gas boiler marginal: '#db6a25'
+    residential rural gas boiler: '#d4722e'
+    residential urban decentral gas boiler: '#cb7a36'
+    services rural gas boiler: '#c4813f'
+    services urban decentral gas boiler: '#ba8947'
+    urban central gas boiler: '#b0904f'
     gas: '#e05b09'
     fossil gas: '#e05b09'
     natural gas: '#e05b09'
+    biogas to gas: '#e36311'
     CCGT: '#a85522'
     CCGT marginal: '#a85522'
     allam: '#B98F76'
@@ -813,6 +824,11 @@ plotting:
     # oil
     oil: '#c9c9c9'
     oil boiler: '#adadad'
+    residential rural oil boiler: '#a9a9a9'
+    services rural oil boiler: '#a5a5a5'
+    residential urban decentral oil boiler: '#a1a1a1'
+    urban central oil boiler: '#9d9d9d'
+    services urban decentral oil boiler: '#999999'
     agriculture machinery oil: '#949494'
     shipping oil: "#808080"
     land transport oil: '#afafaf'
@@ -838,13 +854,20 @@ plotting:
     solid biomass for industry CC: '#47411c'
     solid biomass for industry co2 from atmosphere: '#736412'
     solid biomass for industry co2 to stored: '#47411c'
+    urban central solid biomass CHP: '#9d9042'
+    urban central solid biomass CHP CC: '#6c5d28'
     biomass boiler: '#8A9A5B'
+    residential rural biomass boiler: '#a1a066'
+    residential urban decentral biomass boiler: '#b0b87b'
+    services rural biomass boiler: '#c6cf98'
+    services urban decentral biomass boiler: '#dde5b5'
     biomass to liquid: '#32CD32'
     BioSNG: '#123456'
     # power transmission
     lines: '#6c9459'
     transmission lines: '#6c9459'
     electricity distribution grid: '#97ad8c'
+    low voltage: '#97ad8c'
     # electricity demand
     Electric load: '#110d63'
     electric demand: '#110d63'
@@ -855,24 +878,48 @@ plotting:
     # battery + EVs
     battery: '#ace37f'
     battery storage: '#ace37f'
+    battery charger: '#88a75b'
+    battery discharger: '#5d4e29'
     home battery: '#80c944'
     home battery storage: '#80c944'
+    home battery charger: '#5e8032'
+    home battery discharger: '#3c5221'
     BEV charger: '#baf238'
     V2G: '#e5ffa8'
     land transport EV: '#baf238'
     Li ion: '#baf238'
     # hot water storage
     water tanks: '#e69487'
+    residential rural water tanks: '#f7b7a3'
+    services rural water tanks: '#f3afa3'
+    residential urban decentral water tanks: '#f2b2a3'
+    services urban decentral water tanks: '#f1b4a4'
+    urban central water tanks: '#e9977d'
     hot water storage: '#e69487'
-    hot water charging: '#e69487'
+    hot water charging: '#e8998b'
-    hot water discharging: '#e69487'
+    urban central water tanks charger: '#b57a67'
+    residential rural water tanks charger: '#b4887c'
+    residential urban decentral water tanks charger: '#b39995'
+    services rural water tanks charger: '#b3abb0'
+    services urban decentral water tanks charger: '#b3becc'
+    hot water discharging: '#e99c8e'
+    urban central water tanks discharger: '#b9816e'
+    residential rural water tanks discharger: '#ba9685'
+    residential urban decentral water tanks discharger: '#baac9e'
+    services rural water tanks discharger: '#bbc2b8'
+    services urban decentral water tanks discharger: '#bdd8d3'
     # heat demand
     Heat load: '#cc1f1f'
     heat: '#cc1f1f'
     heat demand: '#cc1f1f'
     rural heat: '#ff5c5c'
+    residential rural heat: '#ff7c7c'
+    services rural heat: '#ff9c9c'
     central heat: '#cc1f1f'
+    urban central heat: '#d15959'
     decentral heat: '#750606'
+    residential urban decentral heat: '#a33c3c'
+    services urban decentral heat: '#cc1f1f'
     low-temperature heat for industry: '#8f2727'
     process heat: '#ff0000'
     agriculture heat: '#d9a5a5'
@@ -880,14 +927,26 @@ plotting:
     heat pumps: '#2fb537'
     heat pump: '#2fb537'
     air heat pump: '#36eb41'
+    residential urban decentral air heat pump: '#48f74f'
+    services urban decentral air heat pump: '#5af95d'
+    urban central air heat pump: '#6cfb6b'
     ground heat pump: '#2fb537'
+    residential rural ground heat pump: '#48f74f'
+    services rural ground heat pump: '#5af95d'
     Ambient: '#98eb9d'
     CHP: '#8a5751'
+    urban central gas CHP: '#8d5e56'
     CHP CC: '#634643'
+    urban central gas CHP CC: '#6e4e4c'
     CHP heat: '#8a5751'
     CHP electric: '#8a5751'
     district heating: '#e8beac'
     resistive heater: '#d8f9b8'
+    residential rural resistive heater: '#bef5b5'
+    residential urban decentral resistive heater: '#b2f1a9'
+    services rural resistive heater: '#a5ed9d'
+    services urban decentral resistive heater: '#98e991'
+    urban central resistive heater: '#8cdf85'
     retrofitting: '#8487e8'
     building retrofitting: '#8487e8'
     # hydrogen
@@ -899,13 +958,16 @@ plotting:
     SMR CC: '#4f1745'
     H2 liquefaction: '#d647bd'
     hydrogen storage: '#bf13a0'
+    H2 Store: '#bf13a0'
     H2 storage: '#bf13a0'
     land transport fuel cell: '#6b3161'
     H2 pipeline: '#f081dc'
     H2 pipeline retrofitted: '#ba99b5'
     H2 Fuel Cell: '#c251ae'
+    H2 fuel cell: '#c251ae'
     H2 turbine: '#991f83'
     H2 Electrolysis: '#ff29d9'
+    H2 electrolysis: '#ff29d9'
     # ammonia
     NH3: '#46caf0'
     ammonia: '#46caf0'
@@ -954,9 +1016,11 @@ plotting:
     waste: '#e3d37d'
     other: '#000000'
     geothermal: '#ba91b1'
+    AC: "#70af1d"
     AC-AC: "#70af1d"
     AC line: "#70af1d"
     links: "#8a1caf"
     HVDC links: "#8a1caf"
+    DC: "#8a1caf"
     DC-DC: "#8a1caf"
     DC link: "#8a1caf"

doc/release_notes.rst

@@ -24,12 +24,17 @@ Upcoming Release
   hydrogen fuel cell. Add switches for both re-electrification options under
   ``sector: hydrogen_turbine:`` and ``sector: hydrogen_fuel_cell:``.
 
+* A new function named ``sanitize_carrier`` ensures that all unique carrier names are present in the network's carriers attribute, and adds nice names and colors for each carrier according to the provided configuration dictionary.
+
+* Additional tech_color are added to include previously unlisted carriers.
+
 * Remove ``vresutils`` dependency.
 
 * Add option to include a piecewise linear approximation of transmission losses,
   e.g. by setting ``solving: options: transmission_losses: 2`` for an
   approximation with two tangents.
 
+
 PyPSA-Eur 0.8.0 (18th March 2023)
 =================================
 

scripts/add_electricity.py

@@ -123,18 +123,61 @@ def calculate_annuity(n, r):
         return 1 / n
 
 
-def _add_missing_carriers_from_costs(n, costs, carriers):
+def sanitize_carriers(n, config):
-    missing_carriers = pd.Index(carriers).difference(n.carriers.index)
+    """
-    if missing_carriers.empty:
+    Sanitize the carrier information in a PyPSA Network object.
-        return
 
-    emissions_cols = (
+    The function ensures that all unique carrier names are present in the network's
-        costs.columns.to_series().loc[lambda s: s.str.endswith("_emissions")].values
+    carriers attribute, and adds nice names and colors for each carrier according
+    to the provided configuration dictionary.
+
+    Parameters
+    ----------
+    n : pypsa.Network
+        A PyPSA Network object that represents an electrical power system.
+    config : dict
+        A dictionary containing configuration information, specifically the
+        "plotting" key with "nice_names" and "tech_colors" keys for carriers.
+
+    Returns
+    -------
+    None
+        The function modifies the 'n' PyPSA Network object in-place, updating the
+        carriers attribute with nice names and colors.
+
+    Warnings
+    --------
+    Raises a warning if any carrier's "tech_colors" are not defined in the config dictionary.
+    """
+
+    for c in n.iterate_components():
+        if "carrier" in c.df:
+            missing_carrier = set(c.df.carrier.unique()) - set(n.carriers.index) - {""}
+            if len(missing_carrier):
+                n.madd("Carrier", missing_carrier)
+
+    carrier_i = n.carriers.index
+    nice_names = (
+        pd.Series(config["plotting"]["nice_names"])
+        .reindex(carrier_i)
+        .fillna(carrier_i.to_series().str.title())
     )
-    suptechs = missing_carriers.str.split("-").str[0]
+    n.carriers["nice_name"] = n.carriers.nice_name.where(
-    emissions = costs.loc[suptechs, emissions_cols].fillna(0.0)
+        n.carriers.nice_name != "", nice_names
-    emissions.index = missing_carriers
+    )
-    n.import_components_from_dataframe(emissions, "Carrier")
+    colors = pd.Series(config["plotting"]["tech_colors"]).reindex(carrier_i)
+    if colors.isna().any():
+        missing_i = list(colors.index[colors.isna()])
+        logger.warning(f"tech_colors for carriers {missing_i} not defined in config.")
+    n.carriers["color"] = n.carriers.color.where(n.carriers.color != "", colors)
+
+
+def add_co2_emissions(n, costs, carriers):
+    """
+    Add CO2 emissions to the network's carriers attribute.
+    """
+    suptechs = n.carriers.loc[carriers].index.str.split("-").str[0]
+    n.carriers.loc[carriers, "co2_emissions"] = costs.co2_emissions[suptechs].values
 
 
 def load_costs(tech_costs, config, max_hours, Nyears=1.0):
@@ -309,57 +352,56 @@ def update_transmission_costs(n, costs, length_factor=1.0):
 
 
 def attach_wind_and_solar(
-    n, costs, input_profiles, technologies, extendable_carriers, line_length_factor=1
+    n, costs, input_profiles, carriers, extendable_carriers, line_length_factor=1
 ):
-    # TODO: rename tech -> carrier, technologies -> carriers
+    n.madd("Carrier", carriers)
-    _add_missing_carriers_from_costs(n, costs, technologies)
 
-    for tech in technologies:
+    for car in carriers:
-        if tech == "hydro":
+        if car == "hydro":
             continue
 
-        with xr.open_dataset(getattr(input_profiles, "profile_" + tech)) as ds:
+        with xr.open_dataset(getattr(input_profiles, "profile_" + car)) as ds:
             if ds.indexes["bus"].empty:
                 continue
 
-            suptech = tech.split("-", 2)[0]
+            supcar = car.split("-", 2)[0]
-            if suptech == "offwind":
+            if supcar == "offwind":
                 underwater_fraction = ds["underwater_fraction"].to_pandas()
                 connection_cost = (
                     line_length_factor
                     * ds["average_distance"].to_pandas()
                     * (
                         underwater_fraction
-                        * costs.at[tech + "-connection-submarine", "capital_cost"]
+                        * costs.at[car + "-connection-submarine", "capital_cost"]
                         + (1.0 - underwater_fraction)
-                        * costs.at[tech + "-connection-underground", "capital_cost"]
+                        * costs.at[car + "-connection-underground", "capital_cost"]
                     )
                 )
                 capital_cost = (
                     costs.at["offwind", "capital_cost"]
-                    + costs.at[tech + "-station", "capital_cost"]
+                    + costs.at[car + "-station", "capital_cost"]
                     + connection_cost
                 )
                 logger.info(
                     "Added connection cost of {:0.0f}-{:0.0f} Eur/MW/a to {}".format(
-                        connection_cost.min(), connection_cost.max(), tech
+                        connection_cost.min(), connection_cost.max(), car
                     )
                 )
             else:
-                capital_cost = costs.at[tech, "capital_cost"]
+                capital_cost = costs.at[car, "capital_cost"]
 
             n.madd(
                 "Generator",
                 ds.indexes["bus"],
-                " " + tech,
+                " " + car,
                 bus=ds.indexes["bus"],
-                carrier=tech,
+                carrier=car,
-                p_nom_extendable=tech in extendable_carriers["Generator"],
+                p_nom_extendable=car in extendable_carriers["Generator"],
                 p_nom_max=ds["p_nom_max"].to_pandas(),
                 weight=ds["weight"].to_pandas(),
-                marginal_cost=costs.at[suptech, "marginal_cost"],
+                marginal_cost=costs.at[supcar, "marginal_cost"],
                 capital_cost=capital_cost,
-                efficiency=costs.at[suptech, "efficiency"],
+                efficiency=costs.at[supcar, "efficiency"],
                 p_max_pu=ds["profile"].transpose("time", "bus").to_pandas(),
             )
 
@@ -373,8 +415,9 @@ def attach_conventional_generators(
     conventional_params,
     conventional_inputs,
 ):
-    carriers = set(conventional_carriers) | set(extendable_carriers["Generator"])
+    carriers = list(set(conventional_carriers) | set(extendable_carriers["Generator"]))
-    _add_missing_carriers_from_costs(n, costs, carriers)
+    n.madd("Carrier", carriers)
+    add_co2_emissions(n, costs, carriers)
 
     ppl = (
         ppl.query("carrier in @carriers")
@@ -430,7 +473,8 @@ def attach_conventional_generators(
 
 
 def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **params):
-    _add_missing_carriers_from_costs(n, costs, carriers)
+    n.madd("Carrier", carriers)
+    add_co2_emissions(n, costs, carriers)
 
     ppl = (
         ppl.query('carrier == "hydro"')
@@ -562,7 +606,8 @@ def attach_extendable_generators(n, costs, ppl, carriers):
     logger.warning(
         "The function `attach_extendable_generators` is deprecated in v0.5.0."
     )
-    _add_missing_carriers_from_costs(n, costs, carriers)
+    n.madd("Carrier", carriers)
+    add_co2_emissions(n, costs, carriers)
 
     for tech in carriers:
         if tech.startswith("OCGT"):
@@ -693,21 +738,6 @@ def estimate_renewable_capacities(n, year, tech_map, expansion_limit, countries)
             )
 
 
-def add_nice_carrier_names(n, config):
-    carrier_i = n.carriers.index
-    nice_names = (
-        pd.Series(config["plotting"]["nice_names"])
-        .reindex(carrier_i)
-        .fillna(carrier_i.to_series().str.title())
-    )
-    n.carriers["nice_name"] = nice_names
-    colors = pd.Series(config["plotting"]["tech_colors"]).reindex(carrier_i)
-    if colors.isna().any():
-        missing_i = list(colors.index[colors.isna()])
-        logger.warning(f"tech_colors for carriers {missing_i} not defined in config.")
-    n.carriers["color"] = colors
-
-
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
@@ -790,7 +820,7 @@ if __name__ == "__main__":
 
     update_p_nom_max(n)
 
-    add_nice_carrier_names(n, snakemake.config)
+    sanitize_carriers(n, snakemake.config)
 
     n.meta = snakemake.config
     n.export_to_netcdf(snakemake.output[0])

scripts/add_existing_baseyear.py

@@ -22,6 +22,7 @@ import numpy as np
 import pypsa
 import xarray as xr
 from _helpers import override_component_attrs, update_config_with_sector_opts
+from add_electricity import sanitize_carriers
 from prepare_sector_network import cluster_heat_buses, define_spatial, prepare_costs
 
 cc = coco.CountryConverter()
@@ -665,4 +666,6 @@ if __name__ == "__main__":
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
 
+    sanitize_carriers(n, snakemake.config)
+
     n.export_to_netcdf(snakemake.output[0])

scripts/add_extra_components.py

@@ -56,11 +56,7 @@ import numpy as np
 import pandas as pd
 import pypsa
 from _helpers import configure_logging
-from add_electricity import (
+from add_electricity import load_costs, sanitize_carriers
-    _add_missing_carriers_from_costs,
-    add_nice_carrier_names,
-    load_costs,
-)
 
 idx = pd.IndexSlice
 
@@ -70,7 +66,7 @@ logger = logging.getLogger(__name__)
 def attach_storageunits(n, costs, extendable_carriers, max_hours):
     carriers = extendable_carriers["StorageUnit"]
 
-    _add_missing_carriers_from_costs(n, costs, carriers)
+    n.madd("Carrier", carriers)
 
     buses_i = n.buses.index
 
@@ -101,7 +97,7 @@ def attach_storageunits(n, costs, extendable_carriers, max_hours):
 def attach_stores(n, costs, extendable_carriers):
     carriers = extendable_carriers["Store"]
 
-    _add_missing_carriers_from_costs(n, costs, carriers)
+    n.madd("Carrier", carriers)
 
     buses_i = n.buses.index
     bus_sub_dict = {k: n.buses[k].values for k in ["x", "y", "country"]}
@@ -161,6 +157,8 @@ def attach_stores(n, costs, extendable_carriers):
             marginal_cost=costs.at["battery", "marginal_cost"],
         )
 
+        n.madd("Carrier", ["battery charger", "battery discharger"])
+
         n.madd(
             "Link",
             b_buses_i + " charger",
@@ -211,6 +209,8 @@ def attach_hydrogen_pipelines(n, costs, extendable_carriers):
     h2_links.index = h2_links.apply(lambda c: f"H2 pipeline {c.bus0}-{c.bus1}", axis=1)
 
     # add pipelines
+    n.add("Carrier", "H2 pipeline")
+
     n.madd(
         "Link",
         h2_links.index,
@@ -245,7 +245,7 @@ if __name__ == "__main__":
     attach_stores(n, costs, extendable_carriers)
     attach_hydrogen_pipelines(n, costs, extendable_carriers)
 
-    add_nice_carrier_names(n, snakemake.config)
+    sanitize_carriers(n, snakemake.config)
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
     n.export_to_netcdf(snakemake.output[0])

scripts/base_network.py

@@ -714,6 +714,7 @@ def base_network(
     n.name = "PyPSA-Eur"
 
     n.set_snapshots(pd.date_range(freq="h", **config["snapshots"]))
+    n.madd("Carrier", ["AC", "DC"])
 
     n.import_components_from_dataframe(buses, "Bus")
     n.import_components_from_dataframe(lines, "Line")

scripts/build_renewable_profiles.py

@@ -216,8 +216,11 @@ if __name__ == "__main__":
     if correction_factor != 1.0:
         logger.info(f"correction_factor is set as {correction_factor}")
 
-    cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1)
+    if nprocesses > 1:
-    client = Client(cluster, asynchronous=True)
+        # cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1)
+        client = Client(n_workers=2, threads_per_worker=2, memory_limit="1GB")
+    else:
+        client = None
 
     cutout = atlite.Cutout(snakemake.input.cutout)
     regions = gpd.read_file(snakemake.input.regions)

scripts/prepare_network.py

@@ -298,6 +298,8 @@ if __name__ == "__main__":
             break
 
     for o in opts:
+        if "+" not in o:
+            continue
         oo = o.split("+")
         suptechs = map(lambda c: c.split("-", 2)[0], n.carriers.index)
         if oo[0].startswith(tuple(suptechs)):

scripts/prepare_sector_network.py

@@ -22,7 +22,7 @@ from _helpers import (
     override_component_attrs,
     update_config_with_sector_opts,
 )
-from add_electricity import calculate_annuity
+from add_electricity import calculate_annuity, sanitize_carriers
 from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2
 from networkx.algorithms import complement
 from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation
@@ -3425,4 +3425,6 @@ if __name__ == "__main__":
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
 
+    sanitize_carriers(n, snakemake.config)
+
     n.export_to_netcdf(snakemake.output[0])