From ad9a67509f6a2d3f62ae4dd2f5685dcac149af50 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Mon, 24 Apr 2023 05:05:43 +0200
Subject: [PATCH 01/30] Add carrier "AC" and "DC" in base_network.py

---
 scripts/base_network.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/base_network.py b/scripts/base_network.py
index 53b4dca3..ecf21e1f 100644
--- a/scripts/base_network.py
+++ b/scripts/base_network.py
@@ -714,6 +714,7 @@ def base_network(
     n.name = "PyPSA-Eur"
 
     n.set_snapshots(pd.date_range(freq="h", **config["snapshots"]))
+    n.madd("Carrier",["AC","DC"])
 
     n.import_components_from_dataframe(buses, "Bus")
     n.import_components_from_dataframe(lines, "Line")

From e550ba915dbfea1a8a4f48dc2efa18ca55741ad5 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Mon, 24 Apr 2023 14:08:25 +0200
Subject: [PATCH 02/30] Update add_extra_components.py

Note:
- battery charger and battery discharger has no nice name nor plot_color
- H2 electrolysis is similar to H2 Electrolysis in the tech_color
- H2 fuel cell is similar to H2 Fuel cell in the tech_color
- H2 pipeline do have have nice name and tech_color, but the addition of it depends on the link being true in config
---
 scripts/add_extra_components.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py
index 020370e5..0d3fcc6f 100644
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@@ -119,6 +119,8 @@ def attach_stores(n, costs, elec_opts):
             e_cyclic=True,
             capital_cost=costs.at["hydrogen storage underground", "capital_cost"],
         )
+        
+        n.madd("Carrier",["H2 electrolysis","H2 fuel cell"])
 
         n.madd(
             "Link",
@@ -161,6 +163,8 @@ def attach_stores(n, costs, elec_opts):
             capital_cost=costs.at["battery storage", "capital_cost"],
             marginal_cost=costs.at["battery", "marginal_cost"],
         )
+        
+        n.madd("Carrier",["battery charger","battery discharger"])
 
         n.madd(
             "Link",
@@ -213,6 +217,8 @@ def attach_hydrogen_pipelines(n, costs, elec_opts):
     h2_links.index = h2_links.apply(lambda c: f"H2 pipeline {c.bus0}-{c.bus1}", axis=1)
 
     # add pipelines
+    n.add("Carrier","H2 pipeline")
+    
     n.madd(
         "Link",
         h2_links.index,

From 71c1a7fbefffa6e5efdff8c8da8361f5b8065f2c Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Tue, 25 Apr 2023 14:43:04 +0200
Subject: [PATCH 03/30] add the function add_missing_carrier

add_missing_carrier() are applied on python script where new components are added. The function runs ideally after all of the new components are added and before add_nice_carrier_names()
---
 scripts/add_electricity.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 1d32bce1..57a18a73 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -120,6 +120,11 @@ def calculate_annuity(n, r):
     else:
         return 1 / n
 
+def add_missing_carrier(n):
+    components = [n.buses, n.generators, n.lines, n.links, n.storage_units, n.stores]
+    for c in components:
+        missing_carrier = np.setdiff1d(c.carrier.unique(),n.carriers.index)
+        n.madd("Carrier",missing_carrier)
 
 def _add_missing_carriers_from_costs(n, costs, carriers):
     missing_carriers = pd.Index(carriers).difference(n.carriers.index)
@@ -832,7 +837,8 @@ if __name__ == "__main__":
         estimate_renewable_capacities(n, snakemake.config)
 
     update_p_nom_max(n)
-
+    
+    add_missing_carrier(n)
     add_nice_carrier_names(n, snakemake.config)
 
     n.meta = snakemake.config

From 56b6718ba6b103062eaee0ff12680a88d9a7660d Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Tue, 25 Apr 2023 17:04:28 +0200
Subject: [PATCH 04/30] Update add_existing_baseyear.py

---
 scripts/add_existing_baseyear.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py
index 5bc0960d..1cc84a36 100644
--- a/scripts/add_existing_baseyear.py
+++ b/scripts/add_existing_baseyear.py
@@ -23,6 +23,7 @@ import pypsa
 import xarray as xr
 from _helpers import override_component_attrs, update_config_with_sector_opts
 from prepare_sector_network import cluster_heat_buses, define_spatial, prepare_costs
+from add_electricity import add_missing_carrier
 
 cc = coco.CountryConverter()
 
@@ -666,5 +667,7 @@ if __name__ == "__main__":
         cluster_heat_buses(n)
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
+    
+    add_missing_carrier(n)
 
     n.export_to_netcdf(snakemake.output[0])

From d9f9d4853a27d1f3e677dff16fb78ee94dced899 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Tue, 25 Apr 2023 17:06:17 +0200
Subject: [PATCH 05/30] Update add_extra_components.py

---
 scripts/add_extra_components.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py
index 0d3fcc6f..dcd000cd 100644
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@@ -57,6 +57,7 @@ import pandas as pd
 import pypsa
 from _helpers import configure_logging
 from add_electricity import (
+    add_missing_carrier,
     _add_missing_carriers_from_costs,
     add_nice_carrier_names,
     load_costs,
@@ -251,6 +252,8 @@ if __name__ == "__main__":
     attach_storageunits(n, costs, elec_config)
     attach_stores(n, costs, elec_config)
     attach_hydrogen_pipelines(n, costs, elec_config)
+    
+    add_missing_carrier(n)
 
     add_nice_carrier_names(n, snakemake.config)
 

From 756a814c636d8b402e53da74be48af7bb241eb75 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Tue, 25 Apr 2023 17:17:32 +0200
Subject: [PATCH 06/30] Update prepare_sector_network.py

---
 scripts/prepare_sector_network.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 6129d3b5..c0def274 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -29,6 +29,7 @@ from pypsa.geo import haversine_pts
 from pypsa.io import import_components_from_dataframe
 from scipy.stats import beta
 from vresutils.costdata import annuity
+from add_electricity import add_missing_carrier
 
 logger = logging.getLogger(__name__)
 
@@ -3398,5 +3399,7 @@ if __name__ == "__main__":
         cluster_heat_buses(n)
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
+    
+    add_missing_carrier(n)
 
     n.export_to_netcdf(snakemake.output[0])

From 74e9d56adb1ea4c71fdaa764efbb9e69a363e8af Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 25 Apr 2023 15:26:23 +0000
Subject: [PATCH 07/30] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/add_electricity.py        |  8 +++++---
 scripts/add_existing_baseyear.py  |  4 ++--
 scripts/add_extra_components.py   | 16 ++++++++--------
 scripts/base_network.py           |  2 +-
 scripts/prepare_sector_network.py |  4 ++--
 5 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 57a18a73..aa201c84 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -120,11 +120,13 @@ def calculate_annuity(n, r):
     else:
         return 1 / n
 
+
 def add_missing_carrier(n):
     components = [n.buses, n.generators, n.lines, n.links, n.storage_units, n.stores]
     for c in components:
-        missing_carrier = np.setdiff1d(c.carrier.unique(),n.carriers.index)
-        n.madd("Carrier",missing_carrier)
+        missing_carrier = np.setdiff1d(c.carrier.unique(), n.carriers.index)
+        n.madd("Carrier", missing_carrier)
+
 
 def _add_missing_carriers_from_costs(n, costs, carriers):
     missing_carriers = pd.Index(carriers).difference(n.carriers.index)
@@ -837,7 +839,7 @@ if __name__ == "__main__":
         estimate_renewable_capacities(n, snakemake.config)
 
     update_p_nom_max(n)
-    
+
     add_missing_carrier(n)
     add_nice_carrier_names(n, snakemake.config)
 
diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py
index 1cc84a36..a8a07322 100644
--- a/scripts/add_existing_baseyear.py
+++ b/scripts/add_existing_baseyear.py
@@ -22,8 +22,8 @@ import numpy as np
 import pypsa
 import xarray as xr
 from _helpers import override_component_attrs, update_config_with_sector_opts
-from prepare_sector_network import cluster_heat_buses, define_spatial, prepare_costs
 from add_electricity import add_missing_carrier
+from prepare_sector_network import cluster_heat_buses, define_spatial, prepare_costs
 
 cc = coco.CountryConverter()
 
@@ -667,7 +667,7 @@ if __name__ == "__main__":
         cluster_heat_buses(n)
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
-    
+
     add_missing_carrier(n)
 
     n.export_to_netcdf(snakemake.output[0])
diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py
index dcd000cd..88606305 100644
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@@ -57,8 +57,8 @@ import pandas as pd
 import pypsa
 from _helpers import configure_logging
 from add_electricity import (
-    add_missing_carrier,
     _add_missing_carriers_from_costs,
+    add_missing_carrier,
     add_nice_carrier_names,
     load_costs,
 )
@@ -120,8 +120,8 @@ def attach_stores(n, costs, elec_opts):
             e_cyclic=True,
             capital_cost=costs.at["hydrogen storage underground", "capital_cost"],
         )
-        
-        n.madd("Carrier",["H2 electrolysis","H2 fuel cell"])
+
+        n.madd("Carrier", ["H2 electrolysis", "H2 fuel cell"])
 
         n.madd(
             "Link",
@@ -164,8 +164,8 @@ def attach_stores(n, costs, elec_opts):
             capital_cost=costs.at["battery storage", "capital_cost"],
             marginal_cost=costs.at["battery", "marginal_cost"],
         )
-        
-        n.madd("Carrier",["battery charger","battery discharger"])
+
+        n.madd("Carrier", ["battery charger", "battery discharger"])
 
         n.madd(
             "Link",
@@ -218,8 +218,8 @@ def attach_hydrogen_pipelines(n, costs, elec_opts):
     h2_links.index = h2_links.apply(lambda c: f"H2 pipeline {c.bus0}-{c.bus1}", axis=1)
 
     # add pipelines
-    n.add("Carrier","H2 pipeline")
-    
+    n.add("Carrier", "H2 pipeline")
+
     n.madd(
         "Link",
         h2_links.index,
@@ -252,7 +252,7 @@ if __name__ == "__main__":
     attach_storageunits(n, costs, elec_config)
     attach_stores(n, costs, elec_config)
     attach_hydrogen_pipelines(n, costs, elec_config)
-    
+
     add_missing_carrier(n)
 
     add_nice_carrier_names(n, snakemake.config)
diff --git a/scripts/base_network.py b/scripts/base_network.py
index ecf21e1f..87504ce7 100644
--- a/scripts/base_network.py
+++ b/scripts/base_network.py
@@ -714,7 +714,7 @@ def base_network(
     n.name = "PyPSA-Eur"
 
     n.set_snapshots(pd.date_range(freq="h", **config["snapshots"]))
-    n.madd("Carrier",["AC","DC"])
+    n.madd("Carrier", ["AC", "DC"])
 
     n.import_components_from_dataframe(buses, "Bus")
     n.import_components_from_dataframe(lines, "Line")
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index c0def274..a0d2f472 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -22,6 +22,7 @@ from _helpers import (
     override_component_attrs,
     update_config_with_sector_opts,
 )
+from add_electricity import add_missing_carrier
 from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2
 from networkx.algorithms import complement
 from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation
@@ -29,7 +30,6 @@ from pypsa.geo import haversine_pts
 from pypsa.io import import_components_from_dataframe
 from scipy.stats import beta
 from vresutils.costdata import annuity
-from add_electricity import add_missing_carrier
 
 logger = logging.getLogger(__name__)
 
@@ -3399,7 +3399,7 @@ if __name__ == "__main__":
         cluster_heat_buses(n)
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
-    
+
     add_missing_carrier(n)
 
     n.export_to_netcdf(snakemake.output[0])

From df38edab0a06bf10d949ac14a1183648dfcdfe07 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Wed, 26 Apr 2023 04:28:36 +0200
Subject: [PATCH 08/30] fix bug in prepare_network.py

---
 scripts/prepare_network.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index 14a003ae..951cc6c9 100755
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -298,6 +298,8 @@ if __name__ == "__main__":
             break
 
     for o in opts:
+        if "+" not in o:
+            continue
         oo = o.split("+")
         suptechs = map(lambda c: c.split("-", 2)[0], n.carriers.index)
         if oo[0].startswith(tuple(suptechs)):

From e24ca89a05fdb3d33578e3a4e4c0ef3e8e8411fd Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Thu, 27 Apr 2023 17:29:20 +0200
Subject: [PATCH 09/30] Update scripts/add_electricity.py

Co-authored-by: Fabian Hofmann <fab.hof@gmx.de>
---
 scripts/add_electricity.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index aa201c84..348e49af 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -125,7 +125,8 @@ def add_missing_carrier(n):
     components = [n.buses, n.generators, n.lines, n.links, n.storage_units, n.stores]
     for c in components:
         missing_carrier = np.setdiff1d(c.carrier.unique(), n.carriers.index)
-        n.madd("Carrier", missing_carrier)
+        if len(missing_carrier):
+            n.madd("Carrier", missing_carrier)
 
 
 def _add_missing_carriers_from_costs(n, costs, carriers):

From 72b75fd9feeeb137d3ad671419a8e6a574d2a041 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Fri, 28 Apr 2023 03:43:20 +0200
Subject: [PATCH 10/30] merge add_nice_carrier_names with add_missing_carrier
 to become add_missing_carrier_with_nice_names(n, config)

---
 scripts/add_electricity.py        | 33 ++++++++++++++-----------------
 scripts/add_existing_baseyear.py  |  4 ++--
 scripts/add_extra_components.py   |  7 ++-----
 scripts/prepare_sector_network.py |  4 ++--
 4 files changed, 21 insertions(+), 27 deletions(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 348e49af..456cc48c 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -121,13 +121,26 @@ def calculate_annuity(n, r):
         return 1 / n
 
 
-def add_missing_carrier(n):
+def add_missing_carriers_with_nice_names(n,config):
     components = [n.buses, n.generators, n.lines, n.links, n.storage_units, n.stores]
     for c in components:
         missing_carrier = np.setdiff1d(c.carrier.unique(), n.carriers.index)
         if len(missing_carrier):
             n.madd("Carrier", missing_carrier)
 
+    carrier_i = n.carriers.index
+    nice_names = (
+        pd.Series(config["plotting"]["nice_names"])
+        .reindex(carrier_i)
+        .fillna(carrier_i.to_series().str.title())
+    )
+    n.carriers["nice_name"] = nice_names
+    colors = pd.Series(config["plotting"]["tech_colors"]).reindex(carrier_i)
+    if colors.isna().any():
+        missing_i = list(colors.index[colors.isna()])
+        logger.warning(f"tech_colors for carriers {missing_i} not defined in config.")
+    n.carriers["color"] = colors
+
 
 def _add_missing_carriers_from_costs(n, costs, carriers):
     missing_carriers = pd.Index(carriers).difference(n.carriers.index)
@@ -694,21 +707,6 @@ def estimate_renewable_capacities(n, config):
             )
 
 
-def add_nice_carrier_names(n, config):
-    carrier_i = n.carriers.index
-    nice_names = (
-        pd.Series(config["plotting"]["nice_names"])
-        .reindex(carrier_i)
-        .fillna(carrier_i.to_series().str.title())
-    )
-    n.carriers["nice_name"] = nice_names
-    colors = pd.Series(config["plotting"]["tech_colors"]).reindex(carrier_i)
-    if colors.isna().any():
-        missing_i = list(colors.index[colors.isna()])
-        logger.warning(f"tech_colors for carriers {missing_i} not defined in config.")
-    n.carriers["color"] = colors
-
-
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
@@ -841,8 +839,7 @@ if __name__ == "__main__":
 
     update_p_nom_max(n)
 
-    add_missing_carrier(n)
-    add_nice_carrier_names(n, snakemake.config)
+    add_missing_carriers_with_nice_names(n, snakemake.config)
 
     n.meta = snakemake.config
     n.export_to_netcdf(snakemake.output[0])
diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py
index a8a07322..1435e660 100644
--- a/scripts/add_existing_baseyear.py
+++ b/scripts/add_existing_baseyear.py
@@ -22,7 +22,7 @@ import numpy as np
 import pypsa
 import xarray as xr
 from _helpers import override_component_attrs, update_config_with_sector_opts
-from add_electricity import add_missing_carrier
+from add_electricity import add_missing_carriers_with_nice_names
 from prepare_sector_network import cluster_heat_buses, define_spatial, prepare_costs
 
 cc = coco.CountryConverter()
@@ -668,6 +668,6 @@ if __name__ == "__main__":
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
 
-    add_missing_carrier(n)
+    add_missing_carriers_with_nice_names(n, snakemake.config)
 
     n.export_to_netcdf(snakemake.output[0])
diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py
index 88606305..da47b7b7 100644
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@@ -58,8 +58,7 @@ import pypsa
 from _helpers import configure_logging
 from add_electricity import (
     _add_missing_carriers_from_costs,
-    add_missing_carrier,
-    add_nice_carrier_names,
+    add_missing_carriers_with_nice_names,
     load_costs,
 )
 
@@ -253,9 +252,7 @@ if __name__ == "__main__":
     attach_stores(n, costs, elec_config)
     attach_hydrogen_pipelines(n, costs, elec_config)
 
-    add_missing_carrier(n)
-
-    add_nice_carrier_names(n, snakemake.config)
+    add_missing_carriers_with_nice_names(n, snakemake.config)
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
     n.export_to_netcdf(snakemake.output[0])
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index a0d2f472..d952cb69 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -22,7 +22,7 @@ from _helpers import (
     override_component_attrs,
     update_config_with_sector_opts,
 )
-from add_electricity import add_missing_carrier
+from add_electricity import add_missing_carriers_with_nice_names
 from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2
 from networkx.algorithms import complement
 from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation
@@ -3400,6 +3400,6 @@ if __name__ == "__main__":
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
 
-    add_missing_carrier(n)
+    add_missing_carriers_with_nice_names(n, snakemake.config)
 
     n.export_to_netcdf(snakemake.output[0])

From 699a4bd2e8e09035e3b93e0d117ba6874e119a8e Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 28 Apr 2023 01:43:39 +0000
Subject: [PATCH 11/30] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/add_electricity.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 456cc48c..2d92405d 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -121,7 +121,7 @@ def calculate_annuity(n, r):
         return 1 / n
 
 
-def add_missing_carriers_with_nice_names(n,config):
+def add_missing_carriers_with_nice_names(n, config):
     components = [n.buses, n.generators, n.lines, n.links, n.storage_units, n.stores]
     for c in components:
         missing_carrier = np.setdiff1d(c.carrier.unique(), n.carriers.index)

From 4e8bbd67b4d1026c60a7f2befa0d4b0b651d96ed Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Wed, 3 May 2023 13:24:57 +0200
Subject: [PATCH 12/30] use sanitize_carriers to harmonize carrier adjustments

---
 scripts/add_electricity.py          | 104 +++++++++++++++++-----------
 scripts/add_existing_baseyear.py    |   4 +-
 scripts/add_extra_components.py     |  14 ++--
 scripts/build_renewable_profiles.py |   7 +-
 scripts/prepare_sector_network.py   |   4 +-
 5 files changed, 78 insertions(+), 55 deletions(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 2d92405d..98d46c82 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -121,12 +121,38 @@ def calculate_annuity(n, r):
         return 1 / n
 
 
-def add_missing_carriers_with_nice_names(n, config):
-    components = [n.buses, n.generators, n.lines, n.links, n.storage_units, n.stores]
-    for c in components:
-        missing_carrier = np.setdiff1d(c.carrier.unique(), n.carriers.index)
-        if len(missing_carrier):
-            n.madd("Carrier", missing_carrier)
+def sanitize_carriers(n, config):
+    """
+    Sanitize the carrier information in a PyPSA Network object.
+
+    The function ensures that all unique carrier names are present in the network's
+    carriers attribute, and adds nice names and colors for each carrier according
+    to the provided configuration dictionary.
+
+    Parameters
+    ----------
+    n : pypsa.Network
+        A PyPSA Network object that represents an electrical power system.
+    config : dict
+        A dictionary containing configuration information, specifically the
+        "plotting" key with "nice_names" and "tech_colors" keys for carriers.
+
+    Returns
+    -------
+    None
+        The function modifies the 'n' PyPSA Network object in-place, updating the
+        carriers attribute with nice names and colors.
+
+    Warnings
+    --------
+    Raises a warning if any carrier's "tech_colors" are not defined in the config dictionary.
+    """
+
+    for c in n.iterate_components():
+        if "carrier" in c.df:
+            missing_carrier = set(c.df.carrier.unique()) - set(n.carriers.index) - {""}
+            if len(missing_carrier):
+                n.madd("Carrier", missing_carrier)
 
     carrier_i = n.carriers.index
     nice_names = (
@@ -142,18 +168,12 @@ def add_missing_carriers_with_nice_names(n, config):
     n.carriers["color"] = colors
 
 
-def _add_missing_carriers_from_costs(n, costs, carriers):
-    missing_carriers = pd.Index(carriers).difference(n.carriers.index)
-    if missing_carriers.empty:
-        return
-
-    emissions_cols = (
-        costs.columns.to_series().loc[lambda s: s.str.endswith("_emissions")].values
-    )
-    suptechs = missing_carriers.str.split("-").str[0]
-    emissions = costs.loc[suptechs, emissions_cols].fillna(0.0)
-    emissions.index = missing_carriers
-    n.import_components_from_dataframe(emissions, "Carrier")
+def add_co2_emissions(n, costs, carriers):
+    """
+    Add CO2 emissions to the network's carriers attribute.
+    """
+    suptechs = n.carriers.loc[carriers].index.str.split("-").str[0]
+    n.carriers.loc[carriers, "co2_emissions"] = costs.co2_emissions[suptechs].values
 
 
 def load_costs(tech_costs, config, elec_config, Nyears=1.0):
@@ -316,57 +336,56 @@ def update_transmission_costs(n, costs, length_factor=1.0):
 
 
 def attach_wind_and_solar(
-    n, costs, input_profiles, technologies, extendable_carriers, line_length_factor=1
+    n, costs, input_profiles, carriers, extendable_carriers, line_length_factor=1
 ):
-    # TODO: rename tech -> carrier, technologies -> carriers
-    _add_missing_carriers_from_costs(n, costs, technologies)
+    n.madd("Carrier", carriers)
 
-    for tech in technologies:
-        if tech == "hydro":
+    for car in carriers:
+        if car == "hydro":
             continue
 
-        with xr.open_dataset(getattr(input_profiles, "profile_" + tech)) as ds:
+        with xr.open_dataset(getattr(input_profiles, "profile_" + car)) as ds:
             if ds.indexes["bus"].empty:
                 continue
 
-            suptech = tech.split("-", 2)[0]
-            if suptech == "offwind":
+            supcar = car.split("-", 2)[0]
+            if supcar == "offwind":
                 underwater_fraction = ds["underwater_fraction"].to_pandas()
                 connection_cost = (
                     line_length_factor
                     * ds["average_distance"].to_pandas()
                     * (
                         underwater_fraction
-                        * costs.at[tech + "-connection-submarine", "capital_cost"]
+                        * costs.at[car + "-connection-submarine", "capital_cost"]
                         + (1.0 - underwater_fraction)
-                        * costs.at[tech + "-connection-underground", "capital_cost"]
+                        * costs.at[car + "-connection-underground", "capital_cost"]
                     )
                 )
                 capital_cost = (
                     costs.at["offwind", "capital_cost"]
-                    + costs.at[tech + "-station", "capital_cost"]
+                    + costs.at[car + "-station", "capital_cost"]
                     + connection_cost
                 )
                 logger.info(
                     "Added connection cost of {:0.0f}-{:0.0f} Eur/MW/a to {}".format(
-                        connection_cost.min(), connection_cost.max(), tech
+                        connection_cost.min(), connection_cost.max(), car
                     )
                 )
             else:
-                capital_cost = costs.at[tech, "capital_cost"]
+                capital_cost = costs.at[car, "capital_cost"]
 
             n.madd(
                 "Generator",
                 ds.indexes["bus"],
-                " " + tech,
+                " " + car,
                 bus=ds.indexes["bus"],
-                carrier=tech,
-                p_nom_extendable=tech in extendable_carriers["Generator"],
+                carrier=car,
+                p_nom_extendable=car in extendable_carriers["Generator"],
                 p_nom_max=ds["p_nom_max"].to_pandas(),
                 weight=ds["weight"].to_pandas(),
-                marginal_cost=costs.at[suptech, "marginal_cost"],
+                marginal_cost=costs.at[supcar, "marginal_cost"],
                 capital_cost=capital_cost,
-                efficiency=costs.at[suptech, "efficiency"],
+                efficiency=costs.at[supcar, "efficiency"],
                 p_max_pu=ds["profile"].transpose("time", "bus").to_pandas(),
             )
 
@@ -380,8 +399,9 @@ def attach_conventional_generators(
     conventional_config,
     conventional_inputs,
 ):
-    carriers = set(conventional_carriers) | set(extendable_carriers["Generator"])
-    _add_missing_carriers_from_costs(n, costs, carriers)
+    carriers = list(set(conventional_carriers) | set(extendable_carriers["Generator"]))
+    n.madd("Carrier", carriers)
+    add_co2_emissions(n, costs, carriers)
 
     ppl = (
         ppl.query("carrier in @carriers")
@@ -435,7 +455,8 @@ def attach_conventional_generators(
 
 
 def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **config):
-    _add_missing_carriers_from_costs(n, costs, carriers)
+    n.madd("Carrier", carriers)
+    add_co2_emissions(n, costs, carriers)
 
     ppl = (
         ppl.query('carrier == "hydro"')
@@ -567,7 +588,8 @@ def attach_extendable_generators(n, costs, ppl, carriers):
     logger.warning(
         "The function `attach_extendable_generators` is deprecated in v0.5.0."
     )
-    _add_missing_carriers_from_costs(n, costs, carriers)
+    n.madd("Carrier", carriers)
+    add_co2_emissions(n, costs, carriers)
 
     for tech in carriers:
         if tech.startswith("OCGT"):
@@ -839,7 +861,7 @@ if __name__ == "__main__":
 
     update_p_nom_max(n)
 
-    add_missing_carriers_with_nice_names(n, snakemake.config)
+    sanitize_carriers(n, snakemake.config)
 
     n.meta = snakemake.config
     n.export_to_netcdf(snakemake.output[0])
diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py
index 1435e660..6999a16b 100644
--- a/scripts/add_existing_baseyear.py
+++ b/scripts/add_existing_baseyear.py
@@ -22,7 +22,7 @@ import numpy as np
 import pypsa
 import xarray as xr
 from _helpers import override_component_attrs, update_config_with_sector_opts
-from add_electricity import add_missing_carriers_with_nice_names
+from add_electricity import sanitize_carriers
 from prepare_sector_network import cluster_heat_buses, define_spatial, prepare_costs
 
 cc = coco.CountryConverter()
@@ -668,6 +668,6 @@ if __name__ == "__main__":
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
 
-    add_missing_carriers_with_nice_names(n, snakemake.config)
+    sanitize_carriers(n, snakemake.config)
 
     n.export_to_netcdf(snakemake.output[0])
diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py
index da47b7b7..e3cc12ec 100644
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@@ -56,11 +56,7 @@ import numpy as np
 import pandas as pd
 import pypsa
 from _helpers import configure_logging
-from add_electricity import (
-    _add_missing_carriers_from_costs,
-    add_missing_carriers_with_nice_names,
-    load_costs,
-)
+from add_electricity import load_costs, sanitize_carriers
 
 idx = pd.IndexSlice
 
@@ -71,7 +67,7 @@ def attach_storageunits(n, costs, elec_opts):
     carriers = elec_opts["extendable_carriers"]["StorageUnit"]
     max_hours = elec_opts["max_hours"]
 
-    _add_missing_carriers_from_costs(n, costs, carriers)
+    n.madd("Carrier", carriers)
 
     buses_i = n.buses.index
 
@@ -102,7 +98,7 @@ def attach_storageunits(n, costs, elec_opts):
 def attach_stores(n, costs, elec_opts):
     carriers = elec_opts["extendable_carriers"]["Store"]
 
-    _add_missing_carriers_from_costs(n, costs, carriers)
+    n.madd("Carrier", carriers)
 
     buses_i = n.buses.index
     bus_sub_dict = {k: n.buses[k].values for k in ["x", "y", "country"]}
@@ -204,6 +200,8 @@ def attach_hydrogen_pipelines(n, costs, elec_opts):
         "`config.yaml` at `electricity: extendable_carriers: Store:`."
     )
 
+    n.add("Carrier", "H2 pipeline")
+
     # determine bus pairs
     attrs = ["bus0", "bus1", "length"]
     candidates = pd.concat(
@@ -252,7 +250,7 @@ if __name__ == "__main__":
     attach_stores(n, costs, elec_config)
     attach_hydrogen_pipelines(n, costs, elec_config)
 
-    add_missing_carriers_with_nice_names(n, snakemake.config)
+    sanitize_carriers(n, snakemake.config)
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
     n.export_to_netcdf(snakemake.output[0])
diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py
index c0288aee..086e4e39 100644
--- a/scripts/build_renewable_profiles.py
+++ b/scripts/build_renewable_profiles.py
@@ -216,8 +216,11 @@ if __name__ == "__main__":
     if correction_factor != 1.0:
         logger.info(f"correction_factor is set as {correction_factor}")
 
-    cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1)
-    client = Client(cluster, asynchronous=True)
+    if nprocesses > 1:
+        # cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1)
+        client = Client(n_workers=2, threads_per_worker=2, memory_limit="1GB")
+    else:
+        client = None
 
     cutout = atlite.Cutout(snakemake.input.cutout)
     regions = gpd.read_file(snakemake.input.regions)
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index d952cb69..bd82e46e 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -22,7 +22,7 @@ from _helpers import (
     override_component_attrs,
     update_config_with_sector_opts,
 )
-from add_electricity import add_missing_carriers_with_nice_names
+from add_electricity import sanitize_carriers
 from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2
 from networkx.algorithms import complement
 from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation
@@ -3400,6 +3400,6 @@ if __name__ == "__main__":
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
 
-    add_missing_carriers_with_nice_names(n, snakemake.config)
+    sanitize_carriers(n, snakemake.config)
 
     n.export_to_netcdf(snakemake.output[0])

From 13b4327e0163e58a1a3edf4933dfbdf9e60f5d10 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Wed, 3 May 2023 14:00:10 +0200
Subject: [PATCH 13/30] add_extra_components: revert carrier addition

---
 scripts/add_extra_components.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py
index e3cc12ec..926d3906 100644
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@@ -200,8 +200,6 @@ def attach_hydrogen_pipelines(n, costs, elec_opts):
         "`config.yaml` at `electricity: extendable_carriers: Store:`."
     )
 
-    n.add("Carrier", "H2 pipeline")
-
     # determine bus pairs
     attrs = ["bus0", "bus1", "length"]
     candidates = pd.concat(

From a28f2c18112fc902b708b539fbf9e11e4530b4af Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Fri, 5 May 2023 00:40:34 +0200
Subject: [PATCH 14/30] add tech_colors for previously missing carrier

---
 config/config.default.yaml | 60 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 58 insertions(+), 2 deletions(-)

diff --git a/config/config.default.yaml b/config/config.default.yaml
index 10e6a6ed..ae3ae655 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -790,6 +790,11 @@ plotting:
     solar: "#f9d002"
     solar PV: "#f9d002"
     solar thermal: '#ffbf2b'
+    residential rural solar thermal: '#f1c069'
+    services rural solar thermal: '#eabf61'
+    residential urban decentral solar thermal: '#e5bc5a'
+    services urban decentral solar thermal: '#dfb953'
+    urban central solar thermal: '#d7b24c'
     solar rooftop: '#ffea80'
     # gas
     OCGT: '#e0986c'
@@ -798,9 +803,15 @@ plotting:
     gas boiler: '#db6a25'
     gas boilers: '#db6a25'
     gas boiler marginal: '#db6a25'
+    residential rural gas boiler: '#d4722e'
+    residential urban decentral gas boiler: '#cb7a36'
+    services rural gas boiler: '#c4813f'
+    services urban decentral gas boiler: '#ba8947'
+    urban central gas boiler: '#b0904f'
     gas: '#e05b09'
     fossil gas: '#e05b09'
     natural gas: '#e05b09'
+    biogas to gas: '#e36311'
     CCGT: '#a85522'
     CCGT marginal: '#a85522'
     allam: '#B98F76'
@@ -838,13 +849,20 @@ plotting:
     solid biomass for industry CC: '#47411c'
     solid biomass for industry co2 from atmosphere: '#736412'
     solid biomass for industry co2 to stored: '#47411c'
+    urban central solid biomass CHP: '#9d9042'
+    urban central solid biomass CHP CC: '#6c5d28'
     biomass boiler: '#8A9A5B'
+    residential rural biomass boiler: '#a1a066'
+    residential urban decentral biomass boiler: '#b0b87b'
+    services rural biomass boiler: '#c6cf98'
+    services urban decentral biomass boiler: '#dde5b5'
     biomass to liquid: '#32CD32'
     BioSNG: '#123456'
     # power transmission
     lines: '#6c9459'
     transmission lines: '#6c9459'
     electricity distribution grid: '#97ad8c'
+    low voltage: '#97ad8c'
     # electricity demand
     Electric load: '#110d63'
     electric demand: '#110d63'
@@ -855,24 +873,48 @@ plotting:
     # battery + EVs
     battery: '#ace37f'
     battery storage: '#ace37f'
+    battery charger: '#88a75b'
+    battery discharger: '#5d4e29'
     home battery: '#80c944'
     home battery storage: '#80c944'
+    home battery charger: '#5e8032'
+    home battery discharger: '#3c5221'
     BEV charger: '#baf238'
     V2G: '#e5ffa8'
     land transport EV: '#baf238'
     Li ion: '#baf238'
     # hot water storage
     water tanks: '#e69487'
+    residential rural water tanks: '#f7b7a3'
+    services rural water tanks: '#f3afa3'
+    residential urban decentral water tanks: '#f2b2a3'
+    services urban decentral water tanks: '#f1b4a4'
+    urban central water tanks: '#e9977d'
     hot water storage: '#e69487'
-    hot water charging: '#e69487'
-    hot water discharging: '#e69487'
+    hot water charging: '#e8998b'
+    urban central water tanks charger: '#b57a67'
+    residential rural water tanks charger: '#b4887c'
+    residential urban decentral water tanks charger: '#b39995'
+    services rural water tanks charger: '#b3abb0'
+    services urban decentral water tanks charger: '#b3becc'
+    hot water discharging: '#e99c8e'
+    urban central water tanks discharger: '#b9816e'
+    residential rural water tanks discharger: '#ba9685'
+    residential urban decentral water tanks discharger: '#baac9e'
+    services rural water tanks discharger: '#bbc2b8'
+    services urban decentral water tanks discharger: '#bdd8d3'
     # heat demand
     Heat load: '#cc1f1f'
     heat: '#cc1f1f'
     heat demand: '#cc1f1f'
     rural heat: '#ff5c5c'
+    residential rural heat: '#ff7c7c'
+    services rural heat: '#ff9c9c'
     central heat: '#cc1f1f'
+    urban central heat: '#d15959'
     decentral heat: '#750606'
+    residential urban decentral heat: '#a33c3c'
+    services urban decentral heat: '#cc1f1f'
     low-temperature heat for industry: '#8f2727'
     process heat: '#ff0000'
     agriculture heat: '#d9a5a5'
@@ -880,14 +922,26 @@ plotting:
     heat pumps: '#2fb537'
     heat pump: '#2fb537'
     air heat pump: '#36eb41'
+    residential urban decentral air heat pump: '#48f74f'
+    services urban decentral air heat pump: '#5af95d'
+    urban central air heat pump: '#6cfb6b'
     ground heat pump: '#2fb537'
+    residential rural ground heat pump: '#48f74f'
+    services rural ground heat pump: '#5af95d'
     Ambient: '#98eb9d'
     CHP: '#8a5751'
+    urban central gas CHP: '#8d5e56'
     CHP CC: '#634643'
+    urban central gas CHP CC: '#6e4e4c'
     CHP heat: '#8a5751'
     CHP electric: '#8a5751'
     district heating: '#e8beac'
     resistive heater: '#d8f9b8'
+    residential rural resistive heater: '#bef5b5'
+    residential urban decentral resistive heater: '#b2f1a9'
+    services rural resistive heater: '#a5ed9d'
+    services urban decentral resistive heater: '#98e991'
+    urban central resistive heater: '#8cdf85'
     retrofitting: '#8487e8'
     building retrofitting: '#8487e8'
     # hydrogen
@@ -899,6 +953,7 @@ plotting:
     SMR CC: '#4f1745'
     H2 liquefaction: '#d647bd'
     hydrogen storage: '#bf13a0'
+    H2 Store: '#bf13a0'
     H2 storage: '#bf13a0'
     land transport fuel cell: '#6b3161'
     H2 pipeline: '#f081dc'
@@ -954,6 +1009,7 @@ plotting:
     waste: '#e3d37d'
     other: '#000000'
     geothermal: '#ba91b1'
+    AC: "#70af1d"
     AC-AC: "#70af1d"
     AC line: "#70af1d"
     links: "#8a1caf"

From 5603951946da59ce85e59e981e705858cf82b308 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Fri, 5 May 2023 11:32:16 +0200
Subject: [PATCH 15/30] fixed little things remaining

---
 config/config.default.yaml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/config/config.default.yaml b/config/config.default.yaml
index ae3ae655..1fe6adde 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -824,6 +824,11 @@ plotting:
     # oil
     oil: '#c9c9c9'
     oil boiler: '#adadad'
+    residential rural oil boiler: '#a9a9a9'
+    services rural oil boiler: '#a5a5a5'
+    residential urban decentral oil boiler: '#a1a1a1'
+    urban central oil boiler: '#9d9d9d'
+    services urban decentral oil boiler: '#999999'
     agriculture machinery oil: '#949494'
     shipping oil: "#808080"
     land transport oil: '#afafaf'
@@ -959,8 +964,10 @@ plotting:
     H2 pipeline: '#f081dc'
     H2 pipeline retrofitted: '#ba99b5'
     H2 Fuel Cell: '#c251ae'
+    H2 fuel cell: '#c251ae'
     H2 turbine: '#991f83'
     H2 Electrolysis: '#ff29d9'
+    H2 electrolysis: '#ff29d9'
     # ammonia
     NH3: '#46caf0'
     ammonia: '#46caf0'
@@ -1014,5 +1021,6 @@ plotting:
     AC line: "#70af1d"
     links: "#8a1caf"
     HVDC links: "#8a1caf"
+    DC: "#8a1caf"
     DC-DC: "#8a1caf"
     DC link: "#8a1caf"

From f6597f43a0417e6074da6125898a386eedcb9767 Mon Sep 17 00:00:00 2001
From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com>
Date: Fri, 5 May 2023 12:16:16 +0200
Subject: [PATCH 16/30] Update release_notes.rst

---
 doc/release_notes.rst | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index 9be10a4a..e1550a93 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -17,11 +17,13 @@ Upcoming Release
 
 * Renamed script file from PyPSA-EUR ``build_load_data`` to ``build_electricity_demand`` and ``retrieve_load_data`` to ``retrieve_electricity_demand``.
 
-
-
 * Add plain hydrogen turbine as additional re-electrification option besides
   hydrogen fuel cell. Add switches for both re-electrification options under
   ``sector: hydrogen_turbine:`` and ``sector: hydrogen_fuel_cell:``.
+  
+* A new function named ``sanitize_carrier`` ensures that all unique carrier names are present in the network's carriers attribute, and adds nice names and colors for each carrier according to the provided configuration dictionary.
+
+* Additional tech_color are added to include previously unlisted carriers.
 
 PyPSA-Eur 0.8.0 (18th March 2023)
 =================================

From b5f5b35f41208af238d6f6bc2bdb2a4fd1f71110 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 5 May 2023 10:16:32 +0000
Subject: [PATCH 17/30] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 doc/release_notes.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index e1550a93..29aff48e 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -20,7 +20,7 @@ Upcoming Release
 * Add plain hydrogen turbine as additional re-electrification option besides
   hydrogen fuel cell. Add switches for both re-electrification options under
   ``sector: hydrogen_turbine:`` and ``sector: hydrogen_fuel_cell:``.
-  
+
 * A new function named ``sanitize_carrier`` ensures that all unique carrier names are present in the network's carriers attribute, and adds nice names and colors for each carrier according to the provided configuration dictionary.
 
 * Additional tech_color are added to include previously unlisted carriers.

From e431a9675cd7da4b229efe95f92ba44e73ff87a9 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 15 May 2023 13:02:39 +0000
Subject: [PATCH 18/30] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 doc/release_notes.rst             | 4 ++--
 scripts/prepare_sector_network.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index be06170a..06e2891e 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -32,8 +32,8 @@ Upcoming Release
 * Add option to include a piecewise linear approximation of transmission losses,
   e.g. by setting ``solving: options: transmission_losses: 2`` for an
   approximation with two tangents.
-  
-  
+
+
 PyPSA-Eur 0.8.0 (18th March 2023)
 =================================
 
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index c19011b8..4fe9f389 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -22,7 +22,7 @@ from _helpers import (
     override_component_attrs,
     update_config_with_sector_opts,
 )
-from add_electricity import sanitize_carriers, calculate_annuity
+from add_electricity import calculate_annuity, sanitize_carriers
 from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2
 from networkx.algorithms import complement
 from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation

From 2f3005e55512cee79639c491c6244a24fdbc3b89 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Thu, 29 Jun 2023 15:37:29 +0200
Subject: [PATCH 19/30] use pypsa.clustering.spatial namespace

---
 scripts/_helpers.py         | 2 +-
 scripts/cluster_network.py  | 6 +++---
 scripts/simplify_network.py | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/scripts/_helpers.py b/scripts/_helpers.py
index 512cb87a..0e1d2327 100644
--- a/scripts/_helpers.py
+++ b/scripts/_helpers.py
@@ -283,7 +283,7 @@ def get_aggregation_strategies(aggregation_strategies):
     # when custom values are specified in the config.
 
     import numpy as np
-    from pypsa.networkclustering import _make_consense
+    from pypsa.clustering.spatial import _make_consense
 
     bus_strategies = dict(country=_make_consense("Bus", "country"))
     bus_strategies.update(aggregation_strategies.get("buses", {}))
diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py
index 52685af2..14741f9e 100644
--- a/scripts/cluster_network.py
+++ b/scripts/cluster_network.py
@@ -89,7 +89,7 @@ Description
     **Is it possible to run the model without the** ``simplify_network`` **rule?**
 
         No, the network clustering methods in the PyPSA module
-        `pypsa.networkclustering <https://github.com/PyPSA/PyPSA/blob/master/pypsa/networkclustering.py>`_
+        `pypsa.clustering.spatial <https://github.com/PyPSA/PyPSA/blob/master/pypsa/clustering/spatial.py>`_
         do not work reliably with multiple voltage levels and transformers.
 
 .. tip::
@@ -134,7 +134,7 @@ import pyomo.environ as po
 import pypsa
 import seaborn as sns
 from _helpers import configure_logging, get_aggregation_strategies, update_p_nom_max
-from pypsa.networkclustering import (
+from pypsa.clustering.spatial import (
     busmap_by_greedy_modularity,
     busmap_by_hac,
     busmap_by_kmeans,
@@ -484,7 +484,7 @@ if __name__ == "__main__":
         # Fast-path if no clustering is necessary
         busmap = n.buses.index.to_series()
         linemap = n.lines.index.to_series()
-        clustering = pypsa.networkclustering.Clustering(
+        clustering = pypsa.clustering.spatial.Clustering(
             n, busmap, linemap, linemap, pd.Series(dtype="O")
         )
     else:
diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 79161760..b4c0829b 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -95,13 +95,13 @@ import scipy as sp
 from _helpers import configure_logging, get_aggregation_strategies, update_p_nom_max
 from add_electricity import load_costs
 from cluster_network import cluster_regions, clustering_for_n_clusters
-from pypsa.io import import_components_from_dataframe, import_series_from_dataframe
-from pypsa.networkclustering import (
+from pypsa.clustering.spatial import (
     aggregategenerators,
     aggregateoneport,
     busmap_by_stubs,
     get_clustering_from_busmap,
 )
+from pypsa.io import import_components_from_dataframe, import_series_from_dataframe
 from scipy.sparse.csgraph import connected_components, dijkstra
 
 logger = logging.getLogger(__name__)

From 8c7c586b0c841a538cd6cbd3aaff9d1df093e875 Mon Sep 17 00:00:00 2001
From: Fabian Hofmann <fab.hof@gmx.de>
Date: Thu, 29 Jun 2023 16:02:03 +0200
Subject: [PATCH 20/30] Update scripts/add_extra_components.py

---
 scripts/add_extra_components.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py
index 006a50a3..14d428e4 100644
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@@ -115,7 +115,6 @@ def attach_stores(n, costs, extendable_carriers):
             capital_cost=costs.at["hydrogen storage underground", "capital_cost"],
         )
 
-        n.madd("Carrier", ["H2 electrolysis", "H2 fuel cell"])
 
         n.madd(
             "Link",

From cf06204e4f8644bef42c5e3eb7d37d329f3b5abf Mon Sep 17 00:00:00 2001
From: Fabian Hofmann <fab.hof@gmx.de>
Date: Thu, 29 Jun 2023 16:02:09 +0200
Subject: [PATCH 21/30] Update scripts/add_electricity.py

---
 scripts/add_electricity.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 8a8f8d00..f5847023 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -167,7 +167,7 @@ def sanitize_carriers(n, config):
     if colors.isna().any():
         missing_i = list(colors.index[colors.isna()])
         logger.warning(f"tech_colors for carriers {missing_i} not defined in config.")
-    n.carriers["color"] = colors
+    n.carriers["color"] = n.carriers.color.where(n.carriers.color != "", colors)
 
 
 def add_co2_emissions(n, costs, carriers):

From a9b0d349f24484aa8a885732eef943c140ad8222 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 29 Jun 2023 14:02:19 +0000
Subject: [PATCH 22/30] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/add_extra_components.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py
index 14d428e4..e00e1e5f 100644
--- a/scripts/add_extra_components.py
+++ b/scripts/add_extra_components.py
@@ -115,7 +115,6 @@ def attach_stores(n, costs, extendable_carriers):
             capital_cost=costs.at["hydrogen storage underground", "capital_cost"],
         )
 
-
         n.madd(
             "Link",
             h2_buses_i + " Electrolysis",

From 3f946dd83d9cf4fd4eb8c64381c7020d115ccb81 Mon Sep 17 00:00:00 2001
From: Fabian Hofmann <fab.hof@gmx.de>
Date: Thu, 29 Jun 2023 16:02:31 +0200
Subject: [PATCH 23/30] Update scripts/add_electricity.py

---
 scripts/add_electricity.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index f5847023..e6c8c01b 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -162,7 +162,7 @@ def sanitize_carriers(n, config):
         .reindex(carrier_i)
         .fillna(carrier_i.to_series().str.title())
     )
-    n.carriers["nice_name"] = nice_names
+    n.carriers["nice_name"] = n.carriers.nice_name.where(n.carriers.nice_name != "", nice_names)
     colors = pd.Series(config["plotting"]["tech_colors"]).reindex(carrier_i)
     if colors.isna().any():
         missing_i = list(colors.index[colors.isna()])

From d84c7ebf6acb0c7d012f7fe3aa40825e47e02fc3 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 29 Jun 2023 14:02:54 +0000
Subject: [PATCH 24/30] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/add_electricity.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index e6c8c01b..4b53fe99 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -162,7 +162,9 @@ def sanitize_carriers(n, config):
         .reindex(carrier_i)
         .fillna(carrier_i.to_series().str.title())
     )
-    n.carriers["nice_name"] = n.carriers.nice_name.where(n.carriers.nice_name != "", nice_names)
+    n.carriers["nice_name"] = n.carriers.nice_name.where(
+        n.carriers.nice_name != "", nice_names
+    )
     colors = pd.Series(config["plotting"]["tech_colors"]).reindex(carrier_i)
     if colors.isna().any():
         missing_i = list(colors.index[colors.isna()])

From 0dd1933057f3a8849114cba122f50881e791358c Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Thu, 29 Jun 2023 16:21:45 +0200
Subject: [PATCH 25/30] solve_network: fix argument for linopy groupby

---
 scripts/solve_network.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index d80e38a0..58f4db72 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -235,8 +235,7 @@ def add_CCL_constraints(n, config):
     p_nom = n.model["Generator-p_nom"]
 
     gens = n.generators.query("p_nom_extendable").rename_axis(index="Generator-ext")
-    grouper = [gens.bus.map(n.buses.country), gens.carrier]
-    grouper = xr.DataArray(pd.MultiIndex.from_arrays(grouper), dims=["Generator-ext"])
+    grouper = pd.concat([gens.bus.map(n.buses.country), gens.carrier])
     lhs = p_nom.groupby(grouper).sum().rename(bus="country")
 
     minimum = xr.DataArray(agg_p_nom_minmax["min"].dropna()).rename(dim_0="group")

From 00ec36a12e43be64ef9001fbdbdb98996f77e899 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Thu, 29 Jun 2023 16:38:56 +0200
Subject: [PATCH 26/30] add_electricity: filter out nan geocoords in kdtree

---
 scripts/add_electricity.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 594988bd..ecd3b70e 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -644,7 +644,7 @@ def attach_OPSD_renewables(n, tech_map):
         buses = n.buses.loc[gens.bus.unique()]
         gens_per_bus = gens.groupby("bus").p_nom.count()
 
-        caps = map_country_bus(df.query("Fueltype == @fueltype"), buses)
+        caps = map_country_bus(df.query("Fueltype == @fueltype and lat == lat"), buses)
         caps = caps.groupby(["bus"]).Capacity.sum()
         caps = caps / gens_per_bus.reindex(caps.index, fill_value=1)
 

From 9e143345bddb840f25da84e7c0f334b92b9434f2 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Thu, 29 Jun 2023 19:24:38 +0200
Subject: [PATCH 27/30] follow up on #686

---
 scripts/build_powerplants.py        | 1 +
 scripts/build_renewable_profiles.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py
index 7f396e1e..f25f111b 100755
--- a/scripts/build_powerplants.py
+++ b/scripts/build_powerplants.py
@@ -149,6 +149,7 @@ if __name__ == "__main__":
         logging.warning(f"No powerplants known in: {', '.join(countries_wo_ppl)}")
 
     substations = n.buses.query("substation_lv")
+    ppl = ppl.dropna(subset=["lat", "lon"])
     ppl = map_country_bus(ppl, substations)
 
     bus_null_b = ppl["bus"].isnull()
diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py
index aa23be3a..864d54dc 100644
--- a/scripts/build_renewable_profiles.py
+++ b/scripts/build_renewable_profiles.py
@@ -204,6 +204,7 @@ if __name__ == "__main__":
 
     nprocesses = int(snakemake.threads)
     noprogress = snakemake.config["run"].get("disable_progressbar", True)
+    noprogress = noprogress or not snakemake.config["atlite"]["show_progress"]
     params = snakemake.params.renewable[snakemake.wildcards.technology]
     resource = params["resource"]  # pv panel params / wind turbine params
     correction_factor = params.get("correction_factor", 1.0)

From 701231fdca3adb465744b46132c9fe57a6fd8346 Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Fri, 30 Jun 2023 11:29:46 +0200
Subject: [PATCH 28/30] build_renewable_profile: fix execution for one thread

---
 rules/build_electricity.smk         | 3 +++
 rules/retrieve.smk                  | 2 --
 scripts/build_renewable_profiles.py | 8 ++++----
 scripts/retrieve_databundle.py      | 2 +-
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index 44ebf404..c7ef86a2 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -59,6 +59,9 @@ rule build_powerplants:
 
 
 rule base_network:
+    params:
+        countries=config["countries"],
+        snapshots=config["snapshots"],
     input:
         eg_buses="data/entsoegridkit/buses.csv",
         eg_lines="data/entsoegridkit/lines.csv",
diff --git a/rules/retrieve.smk b/rules/retrieve.smk
index 4bfbd6c6..0a96406a 100644
--- a/rules/retrieve.smk
+++ b/rules/retrieve.smk
@@ -19,8 +19,6 @@ if config["enable"].get("retrieve_databundle", True):
         datafiles.extend(["natura/Natura2000_end2015.shp", "GEBCO_2014_2D.nc"])
 
     rule retrieve_databundle:
-        params:
-            tutorial=config["tutorial"],
         output:
             expand("data/bundle/{file}", file=datafiles),
         log:
diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py
index 864d54dc..88a43f53 100644
--- a/scripts/build_renewable_profiles.py
+++ b/scripts/build_renewable_profiles.py
@@ -188,7 +188,7 @@ import geopandas as gpd
 import numpy as np
 import xarray as xr
 from _helpers import configure_logging
-from dask.distributed import Client, LocalCluster
+from dask.distributed import Client
 from pypsa.geo import haversine
 from shapely.geometry import LineString
 
@@ -218,8 +218,7 @@ if __name__ == "__main__":
         logger.info(f"correction_factor is set as {correction_factor}")
 
     if nprocesses > 1:
-        # cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1)
-        client = Client(n_workers=2, threads_per_worker=2, memory_limit="1GB")
+        client = Client(n_workers=nprocesses, threads_per_worker=1)
     else:
         client = None
 
@@ -293,7 +292,8 @@ if __name__ == "__main__":
 
     potential = capacity_per_sqkm * availability.sum("bus") * area
     func = getattr(cutout, resource.pop("method"))
-    resource["dask_kwargs"] = {"scheduler": client}
+    if client is not None:
+        resource["dask_kwargs"] = {"scheduler": client}
     capacity_factor = correction_factor * func(capacity_factor=True, **resource)
     layout = capacity_factor * area * capacity_per_sqkm
     profile, capacities = func(
diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py
index c7053bed..75d8519e 100644
--- a/scripts/retrieve_databundle.py
+++ b/scripts/retrieve_databundle.py
@@ -53,7 +53,7 @@ if __name__ == "__main__":
         snakemake
     )  # TODO Make logging compatible with progressbar (see PR #102)
 
-    if snakemake.params.tutorial:
+    if snakemake.config["tutorial"]:
         url = "https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz"
     else:
         url = "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz"

From 16dcad01760aaa5cdba2a07522bbdac247245481 Mon Sep 17 00:00:00 2001
From: euronion <42553970+euronion@users.noreply.github.com>
Date: Fri, 30 Jun 2023 13:25:20 +0200
Subject: [PATCH 29/30] Prevent attempt to re-add existing carriers.

---
 doc/release_notes.rst      |  2 ++
 scripts/add_electricity.py | 19 ++++++++++++-------
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index ddd1483e..727675c2 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -16,6 +16,8 @@ Upcoming Release
 
 * Bugfix: Correct typo in the CPLEX solver configuration in ``config.default.yaml``.
 
+* Bugfix: Error in ``add_electricity`` where carriers were added multiple times to the network, resulting in a non-unique carriers error.
+
 * Renamed script file from PyPSA-EUR ``build_load_data`` to ``build_electricity_demand`` and ``retrieve_load_data`` to ``retrieve_electricity_demand``.
 
 * Fix docs readthedocs built
diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 88860790..0e528644 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -122,6 +122,13 @@ def calculate_annuity(n, r):
     else:
         return 1 / n
 
+def add_missing_carriers(n, carriers):
+    """
+    Function to add missing carriers to the network without raising errors.
+    """
+    missing_carriers = set(carriers) - set(n.carriers.index)
+    if len(missing_carriers) > 0:
+        n.madd("Carrier", missing_carriers)
 
 def sanitize_carriers(n, config):
     """
@@ -152,9 +159,7 @@ def sanitize_carriers(n, config):
 
     for c in n.iterate_components():
         if "carrier" in c.df:
-            missing_carrier = set(c.df.carrier.unique()) - set(n.carriers.index) - {""}
-            if len(missing_carrier):
-                n.madd("Carrier", missing_carrier)
+            add_missing_carriers(n, c.df)
 
     carrier_i = n.carriers.index
     nice_names = (
@@ -354,7 +359,7 @@ def update_transmission_costs(n, costs, length_factor=1.0):
 def attach_wind_and_solar(
     n, costs, input_profiles, carriers, extendable_carriers, line_length_factor=1
 ):
-    n.madd("Carrier", carriers)
+    add_missing_carriers(n, carriers)
 
     for car in carriers:
         if car == "hydro":
@@ -416,7 +421,7 @@ def attach_conventional_generators(
     conventional_inputs,
 ):
     carriers = list(set(conventional_carriers) | set(extendable_carriers["Generator"]))
-    n.madd("Carrier", carriers)
+    add_missing_carriers(n, carriers)
     add_co2_emissions(n, costs, carriers)
 
     ppl = (
@@ -473,7 +478,7 @@ def attach_conventional_generators(
 
 
 def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **params):
-    n.madd("Carrier", carriers)
+    add_missing_carriers(n, carriers)
     add_co2_emissions(n, costs, carriers)
 
     ppl = (
@@ -606,7 +611,7 @@ def attach_extendable_generators(n, costs, ppl, carriers):
     logger.warning(
         "The function `attach_extendable_generators` is deprecated in v0.5.0."
     )
-    n.madd("Carrier", carriers)
+    add_missing_carriers(n, carriers)
     add_co2_emissions(n, costs, carriers)
 
     for tech in carriers:

From e4651ce711d2bd1f53736e3ec0abf77c4b137c04 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 30 Jun 2023 11:28:54 +0000
Subject: [PATCH 30/30] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/add_electricity.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py
index 0e528644..2f6251f0 100755
--- a/scripts/add_electricity.py
+++ b/scripts/add_electricity.py
@@ -122,6 +122,7 @@ def calculate_annuity(n, r):
     else:
         return 1 / n
 
+
 def add_missing_carriers(n, carriers):
     """
     Function to add missing carriers to the network without raising errors.
@@ -130,6 +131,7 @@ def add_missing_carriers(n, carriers):
     if len(missing_carriers) > 0:
         n.madd("Carrier", missing_carriers)
 
+
 def sanitize_carriers(n, config):
     """
     Sanitize the carrier information in a PyPSA Network object.