Merge branch 'master' into pz-max-patch-2

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
 
   # Sort package imports alphabetically
 - repo: https://github.com/PyCQA/isort
-  rev: 5.11.0
+  rev: 5.11.4
   hooks:
   - id: isort
     args: ["--profile", "black", "--filter-files"]
@@ -39,7 +39,7 @@ repos:
 
   # Make docstrings PEP 257 compliant
 - repo: https://github.com/PyCQA/docformatter
-  rev: v1.5.0
+  rev: v1.5.1
   hooks:
   - id: docformatter
     args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"]
@@ -74,7 +74,7 @@ repos:
 
   # Format Snakemake rule / workflow files
 - repo: https://github.com/snakemake/snakefmt
-  rev: v0.7.0
+  rev: v0.8.0
   hooks:
   - id: snakefmt
 

Snakefile

@@ -68,7 +68,7 @@ if config["enable"].get("prepare_links_p_nom", False):
             "logs/" + RDIR + "prepare_links_p_nom.log",
         threads: 1
         resources:
-            mem_mb=500,
+            mem_mb=1500,
         script:
             "scripts/prepare_links_p_nom.py"
 
@@ -167,7 +167,7 @@ rule base_network:
         "benchmarks/" + RDIR + "base_network"
     threads: 1
     resources:
-        mem_mb=500,
+        mem_mb=1500,
     script:
         "scripts/base_network.py"
 
@@ -190,7 +190,7 @@ rule build_shapes:
         "logs/" + RDIR + "build_shapes.log",
     threads: 1
     resources:
-        mem_mb=500,
+        mem_mb=1500,
     script:
         "scripts/build_shapes.py"
 
@@ -657,7 +657,7 @@ rule make_summary:
         + RDIR
         + "make_summary/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.log",
     resources:
-        mem_mb=500,
+        mem_mb=1500,
     script:
         "scripts/make_summary.py"
 
@@ -676,7 +676,7 @@ rule plot_summary:
         + RDIR
         + "plot_summary/{summary}_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}_{ext}.log",
     resources:
-        mem_mb=500,
+        mem_mb=1500,
     script:
         "scripts/plot_summary.py"
 
@@ -706,6 +706,6 @@ rule plot_p_nom_max:
         + RDIR
         + "plot_p_nom_max/elec_s{simpl}_{clusts}_{techs}_{country}_{ext}.log",
     resources:
-        mem_mb=500,
+        mem_mb=1500,
     script:
         "scripts/plot_p_nom_max.py"

config.default.yaml

@@ -86,6 +86,7 @@ electricity:
 
 atlite:
   nprocesses: 4
+  show_progress: false # false saves time
   cutouts:
     # use 'base' to determine geographical bounds and time span from config
     # base:
@@ -262,6 +263,8 @@ clustering:
     algorithm: kmeans # choose from: [hac, kmeans]
     feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc.
     exclude_carriers: []
+    remove_stubs: true
+    remove_stubs_across_borders: true
   cluster_network:
     algorithm: kmeans
     feature: solar+onwind-time

config.tutorial.yaml

@@ -56,6 +56,7 @@ electricity:
 
 atlite:
   nprocesses: 4
+  show_progress: false # false saves time
   cutouts:
     be-03-2013-era5:
       module: era5

data/links_tyndp.csv

@@ -24,3 +24,5 @@ Gridlink,Kingsnorth (UK),Warande (FR),160,,1400,in permitting,,https://tyndp.ent
 NeuConnect,Grain (UK),Fedderwarden (DE),680,,1400,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/309,0.716666666666667,51.44,8.046524,53.562763
 NordBalt,Klaipeda (LT),Nybro (SE),450,,700,built,,https://en.wikipedia.org/wiki/NordBalt,21.256667,55.681667,15.854167,56.767778
 Estlink 1,Harku (EE),Espoo (FI),105,,350,built,,https://en.wikipedia.org/wiki/Estlink,24.560278,59.384722,24.551667,60.203889
+Greenlink,Waterford (IE),Pembroke (UK),,180,500,under construction,,https://tyndp2022-project-platform.azurewebsites.net/projectsheets/transmission/286,-6.987,52.260,-4.986,51.686
+Celtic Interconnector,Aghada (IE),La Martyre (FR),,572,700,under consideration,,https://tyndp2022-project-platform.azurewebsites.net/projectsheets/transmission/107,-8.16642,51.91413,-4.184,48.459

data/parameter_corrections.yaml

@@ -36,12 +36,20 @@ Link:
       "5583": "7428" # bus0 == bus1 to remove link in remove_unconnected_components (Sardinia)
       "13588": "7428" # bus0 == bus1 to remove link in remove_unconnected_components (Sardinia)
       "T23": "6355" # bus0 == bus1 to remove link in remove_unconnected_components (NordBalt)
+      "14815": "5939" # Kainachtal
+      "8706": "6448"
   bus1:
     index:
       "12931": "8152" # BorWin3
       "5582": "2382" # combine link 5583 + 5582 in 5582 (Sardinia)
       "13589": "1349" # combine link 13589 + 13588 in 13589 (Sardinia)
       "14820": "6354" # NordBalt
+      "14810": "6365" # Skagerrak
+      "8708": "6448"
+      "8394": "6695"
+      "14813": "7052"
+      "8009": "5939"
+      "5601": "7052" # Link Sweden - Lübeck
   length:
     index:
       "5582": 26.39 # new length of combined links (sum)
@@ -53,6 +61,7 @@ Line:
   bus0:
     index:
       "14573": "7179" #fix bus-id substation in PT (220/380kV issue)
+      "14756": "8577" # Deeside connection
   v_nom:
     index:
       "14573": 220 # 220/380kV issue of substation in PT

envs/environment.yaml

@@ -10,7 +10,7 @@ dependencies:
 - python>=3.8
 - pip
 
-- pypsa>=0.21.2
+- pypsa>=0.21.3
 - atlite>=0.2.9
 - dask
 
@@ -24,20 +24,20 @@ dependencies:
 - yaml
 - pytables
 - lxml
-- powerplantmatching>=0.5.4
+- powerplantmatching>=0.5.5
-- numpy
+- numpy<1.24
 - pandas
 - geopandas>=0.11.0
 - xarray
 - netcdf4
 - networkx
 - scipy
-- shapely<2.0   # need to address deprecations
+- shapely<2.0
 - progressbar2
 - pyomo
-- matplotlib
+- matplotlib<3.6
 - proj
-- fiona <= 1.18.20    # Till issue https://github.com/Toblerity/Fiona/issues/1085 is not solved
+- fiona
 - country_converter
 
   # Keep in conda environment when calling ipython

scripts/build_shapes.py

@@ -129,6 +129,8 @@ def countries(naturalearth, country_list):
     s = df.set_index("name")["geometry"].map(_simplify_polys)
     if "RS" in country_list:
         s["RS"] = s["RS"].union(s.pop("KV"))
+        # cleanup shape union
+        s["RS"] = Polygon(s["RS"].exterior.coords)
 
     return s
 
@@ -145,7 +147,7 @@ def eez(country_shapes, eez, country_list):
         lambda s: _simplify_polys(s, filterremote=False)
     )
     s = gpd.GeoSeries(
-        {k: v for k, v in s.iteritems() if v.distance(country_shapes[k]) < 1e-3}
+        {k: v for k, v in s.items() if v.distance(country_shapes[k]) < 1e-3}
     )
     s = s.to_frame("geometry")
     s.index.name = "name"

scripts/simplify_network.py

@@ -217,7 +217,7 @@ def _adjust_capital_costs_using_connection_costs(n, connection_costs_to_bus, out
                     tech,
                     ", ".join(
                         "{:.0f} Eur/MW/a for `{}`".format(d, b)
-                        for b, d in costs.iteritems()
+                        for b, d in costs.items()
                     ),
                 )
             )
@@ -369,7 +369,7 @@ def simplify_links(n, costs, config, output, aggregation_strategies=dict()):
             n.mremove("Link", all_links)
 
             static_attrs = n.components["Link"]["attrs"].loc[lambda df: df.static]
-            for attr, default in static_attrs.default.iteritems():
+            for attr, default in static_attrs.default.items():
                 params.setdefault(attr, default)
             n.links.loc[name] = pd.Series(params)
 
@@ -395,7 +395,9 @@ def simplify_links(n, costs, config, output, aggregation_strategies=dict()):
 def remove_stubs(n, costs, config, output, aggregation_strategies=dict()):
     logger.info("Removing stubs")
 
-    busmap = busmap_by_stubs(n)  #  ['country'])
+    across_borders = config["clustering"]["simplify_network"].get("remove_stubs_across_borders", True)
+    matching_attrs = [] if across_borders else ['country']
+    busmap = busmap_by_stubs(n, matching_attrs)
 
     connection_costs_to_bus = _compute_connection_costs_to_bus(n, busmap, costs, config)
 
@@ -530,22 +532,20 @@ if __name__ == "__main__":
         n, technology_costs, snakemake.config, snakemake.output, aggregation_strategies
     )
 
-    n, stub_map = remove_stubs(
+    busmaps = [trafo_map, simplify_links_map]
-        n,
-        technology_costs,
-        snakemake.config,
-        snakemake.output,
-        aggregation_strategies=aggregation_strategies,
-    )
 
-    busmaps = [trafo_map, simplify_links_map, stub_map]
+    cluster_config = snakemake.config["clustering"]["simplify_network"]
+    if cluster_config.get("remove_stubs", True):
+        n, stub_map = remove_stubs(
+            n,
+            technology_costs,
+            snakemake.config,
+            snakemake.output,
+            aggregation_strategies=aggregation_strategies,
+        )
+        busmaps.append(stub_map)
 
-    cluster_config = snakemake.config.get("clustering", {}).get("simplify_network", {})
+    if cluster_config.get("to_substations", False):
-    if (
-        cluster_config.get("clustering", {})
-        .get("simplify_network", {})
-        .get("to_substations", False)
-    ):
         n, substation_map = aggregate_to_substations(n, aggregation_strategies)
         busmaps.append(substation_map)
 

test/config.test1.yaml

@@ -54,6 +54,7 @@ electricity:
 
 atlite:
   nprocesses: 4
+  show_progress: false # false saves time
   cutouts:
     be-03-2013-era5:
       module: era5