Merge branch 'master' into drop-renewables

.github/workflows/ci.yaml

@@ -31,7 +31,7 @@ jobs:
         os:
         - ubuntu-latest
         - macos-latest
-        - windows-latest
+#        - windows-latest
         inhouse:
         - stable
         - master

.pre-commit-config.yaml

@@ -67,7 +67,7 @@ repos:
 
   # Do YAML formatting (before the linter checks it for misses)
 - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
-  rev: v2.12.0
+  rev: v2.13.0
   hooks:
   - id: pretty-format-yaml
     args: [--autofix, --indent, "2", --preserve-quotes]

config/config.default.yaml

@@ -786,7 +786,6 @@ solving:
       PreDual: 0
       GURO_PAR_BARDENSETHRESH: 200
     gurobi-numeric-focus:
-      name: gurobi
       NumericFocus: 3       # Favour numeric stability over speed
       method: 2             # barrier
       crossover: 0          # do not use crossover
@@ -798,7 +797,6 @@ solving:
       threads: 8
       Seed: 123
     gurobi-fallback:        # Use gurobi defaults
-      name: gurobi
       crossover: 0
       method: 2             # barrier
       BarHomogeneous: 1     # Use homogeneous barrier if standard does not converge

data/links_tyndp.csv

@@ -26,3 +26,16 @@ NordBalt,Klaipeda (LT),Nybro (SE),450,,700,built,,https://en.wikipedia.org/wiki/
 Estlink 1,Harku (EE),Espoo (FI),105,,350,built,,https://en.wikipedia.org/wiki/Estlink,24.560278,59.384722,24.551667,60.203889
 Greenlink,Waterford (IE),Pembroke (UK),,180,500,under construction,,https://tyndp2022-project-platform.azurewebsites.net/projectsheets/transmission/286,-6.987,52.260,-4.986,51.686
 Celtic Interconnector,Aghada (IE),La Martyre (FR),,572,700,under consideration,,https://tyndp2022-project-platform.azurewebsites.net/projectsheets/transmission/107,-8.16642,51.91413,-4.184,48.459
+GiLA,Bordeaux (FR),Nantes (FR),,312,640,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,-1.209,46.901,-0.576,44.960
+HG North Tyrrhenian Corridor,Milan (IT),Viterbo (IT),,500,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,9.409,45.553,12.015,42.244
+HG Adriatic Corridor,Ferrara (IT),Foggia (IT),,582,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,11.661,44.855,15.550,41.513
+SAPEI 2,Fioumesanto (IT),Montalto (IT),,390,1000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,8.283,40.790,11.602,42.331
+HG Ionian-Tyrrhenian Corridor,Rossano (IT),Latina (IT),,496,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,16.629,39.568,12.779,41.430
+HG Ionian-Tyrrhenian Corridor 2,Rossano (IT),Catania (IT),,330,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,16.629,39.568,15.049,37.408
+Germany-UK Hybrid Interconnector,Fetteresso (UK),Emden (DE),800,,2000,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,-2.383,56.991,7.207,53.376
+NU-Link Interconnector,Hornsea (UK),Moerdijk (NL),,460,1200,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,-0.261,53.655,4.586,51.661
+APOLLO-LINK,La Farga (ES),La Spezia (IT),,725,2091,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,2.883,42.062,9.884,44.107
+Baltic WindConnector (BWC),Lubmin (DE),Lihula (EE),,960,2000,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,13.686,54.139,23.818,58.675
+High-Voltage Direct Current Interconnector Project Romania-Hungary,Constanta (RO),Albertirsa (HU),,930,2500,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,28.588,44.201,19.584,47.224
+Rhine-Main-Link,Ovelgönne (DE),Marxheim (DE),,433,4000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,8.379,53.315,8.435,50.078
+Green Aegean Interconnector,Arachthos (GR),Ottenhofen (DE),,600,3000,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,20.967,39.185,11.868,48.207

doc/release_notes.rst

@@ -10,6 +10,10 @@ Release Notes
 Upcoming Release
 ================
 
+* bugfix: convert Strings to pathlib.Path objects as input to ConfigSettings
+
+* Allow the use of more solvers in clustering (Xpress, COPT, Gurobi, CPLEX, SCIP, MOSEK).
+
 * Enhanced support for choosing different weather years
   (https://github.com/PyPSA/pypsa-eur/pull/204):
 
@@ -59,6 +63,9 @@ Upcoming Release
 * Removed rule ``copy_config``. Instead, a config file is created for each
   network output of the ``solve_*`` rules, with the same content as ``n.meta``.
 
+* Added new HVDC transmission projects from `TYNDP 2024 draft projects
+  <https://tyndp.entsoe.eu/news/176-pan-european-electricity-transmission-projects-and-33-storage-projects-will-be-assessed-in-tyndp-2024>`_.
+
 * Upgrade to Snakemake v8.5+. This version is the new minimum version required.
   To upgrade an existing environment, run ``conda install -c bioconda
   snakemake-minimal">=8.5"`` and ``pip install snakemake-storage-plugin-http``
@@ -159,7 +166,11 @@ Upcoming Release
 
 * Adapt the disabling of transmission expansion in myopic foresight optimisations when limit is already reached to also handle cost limits.
 
-* Fix duplicated years in `add_land_use_constraint_m`.
+* Fix duplicated years and grouping years reference in `add_land_use_constraint_m`.
+
+* Fix type error with `m` option in `cluster_network`.
+
+* Fix error with `symbol` of `buses` in `simplify_network`.
 
 PyPSA-Eur 0.10.0 (19th February 2024)
 =====================================

rules/postprocess.smk

@@ -44,7 +44,7 @@ if config["foresight"] != "perfect":
         benchmark:
             (
                 RESULTS
-                + "benchmarksplot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
+                + "benchmarks/plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
             )
         conda:
             "../envs/environment.yaml"

rules/solve_perfect.smk

@@ -35,7 +35,8 @@ rule add_existing_baseyear:
         planning_horizons=config["scenario"]["planning_horizons"][0],  #only applies to baseyear
     threads: 1
     resources:
-        mem_mb=2000,
+        mem_mb=config_provider("solving", "mem_mb"),
+        runtime=config_provider("solving", "runtime", default="24h"),
     log:
         logs(
             "add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"

scripts/_helpers.py

@@ -43,10 +43,11 @@ def get_scenarios(run):
     scenario_config = run.get("scenarios", {})
     if run["name"] and scenario_config.get("enable"):
         fn = Path(scenario_config["file"])
-        scenarios = yaml.safe_load(fn.read_text())
-        if run["name"] == "all":
-            run["name"] = list(scenarios.keys())
-        return scenarios
+        if fn.exists():
+            scenarios = yaml.safe_load(fn.read_text())
+            if run["name"] == "all":
+                run["name"] = list(scenarios.keys())
+            return scenarios
     return {}
 
 
@@ -102,20 +103,17 @@ def get_run_path(fn, dir, rdir, shared_resources):
             "add_electricity"
         )
         is_shared = no_relevant_wildcards and no_elec_rule
+        rdir = "" if is_shared else rdir
     elif isinstance(shared_resources, str):
         rdir = shared_resources + "/"
-        is_shared = False
     elif isinstance(shared_resources, bool):
-        is_shared = shared_resources
+        rdir = "" if shared_resources else rdir
     else:
         raise ValueError(
             "shared_resources must be a boolean, str, or 'base' for special handling."
         )
 
-    if is_shared:
-        return f"{dir}{fn}"
-    else:
-        return f"{dir}{rdir}{fn}"
+    return f"{dir}{rdir}{fn}"
 
 
 def path_provider(dir, rdir, shared_resources):
@@ -429,7 +427,7 @@ def mock_snakemake(
             configfiles = [configfiles]
 
         resource_settings = ResourceSettings()
-        config_settings = ConfigSettings(configfiles=configfiles)
+        config_settings = ConfigSettings(configfiles=map(Path, configfiles))
         workflow_settings = WorkflowSettings()
         storage_settings = StorageSettings()
         dag_settings = DAGSettings(rerun_triggers=[])

scripts/build_energy_totals.py

@@ -124,6 +124,17 @@ def build_eurostat(input_eurostat, countries, nprocesses=1, disable_progressbar=
     )
     df = pd.concat([temp, df.loc[~int_avia]])
 
+    # Fill in missing data on "Domestic aviation" for each country.
+    domestic_avia = df.index.get_level_values(4) == "Domestic aviation"
+    for country in countries:
+        slicer = idx[country, :, :, :, "Domestic aviation"]
+        # For the Total and Fossil energy columns, fill in zeros with
+        # the closest non-zero value in the year index.
+        for col in ["Total", "Fossil energy"]:
+            df.loc[slicer, col] = (
+                df.loc[slicer, col].replace(0.0, np.nan).ffill().bfill()
+            )
+
     # Renaming some indices
     index_rename = {
         "Households": "Residential",

scripts/build_sequestration_potentials.py

@@ -44,7 +44,7 @@ if __name__ == "__main__":
 
     cf = snakemake.params.sequestration_potential
 
-    gdf = gpd.read_file(snakemake.input.sequestration_potential[0])
+    gdf = gpd.read_file(snakemake.input.sequestration_potential)
 
     regions = gpd.read_file(snakemake.input.regions_offshore)
     if cf["include_onshore"]:

scripts/cluster_network.py

@@ -231,7 +231,7 @@ def distribute_clusters(n, n_clusters, focus_weights=None, solver_name="scip"):
         .pipe(normed)
     )
 
-    N = n.buses.groupby(["country", "sub_network"]).size()
+    N = n.buses.groupby(["country", "sub_network"]).size()[L.index]
 
     assert (
         n_clusters >= len(N) and n_clusters <= N.sum()
@@ -267,7 +267,7 @@ def distribute_clusters(n, n_clusters, focus_weights=None, solver_name="scip"):
     m.objective = (clusters * clusters - 2 * clusters * L * n_clusters).sum()
     if solver_name == "gurobi":
         logging.getLogger("gurobipy").propagate = False
-    elif solver_name not in ["scip", "cplex"]:
+    elif solver_name not in ["scip", "cplex", "xpress", "copt", "mosek"]:
         logger.info(
             f"The configured solver `{solver_name}` does not support quadratic objectives. Falling back to `scip`."
         )
@@ -454,7 +454,7 @@ if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
 
-        snakemake = mock_snakemake("cluster_network", simpl="", clusters="5")
+        snakemake = mock_snakemake("cluster_network", simpl="", clusters="40")
     configure_logging(snakemake)
     set_scenario_config(snakemake)
 
@@ -471,7 +471,7 @@ if __name__ == "__main__":
     conventional_carriers = set(params.conventional_carriers)
     if snakemake.wildcards.clusters.endswith("m"):
         n_clusters = int(snakemake.wildcards.clusters[:-1])
-        aggregate_carriers = params.conventional_carriers & aggregate_carriers
+        aggregate_carriers = conventional_carriers & aggregate_carriers
     elif snakemake.wildcards.clusters.endswith("c"):
         n_clusters = int(snakemake.wildcards.clusters[:-1])
         aggregate_carriers = aggregate_carriers - conventional_carriers

scripts/simplify_network.py

@@ -594,18 +594,6 @@ if __name__ == "__main__":
             )
             busmaps.append(busmap_hac)
 
-    if snakemake.wildcards.simpl:
-        n, cluster_map = cluster(
-            n,
-            int(snakemake.wildcards.simpl),
-            params.focus_weights,
-            solver_name,
-            params.simplify_network["algorithm"],
-            params.simplify_network["feature"],
-            params.aggregation_strategies,
-        )
-        busmaps.append(cluster_map)
-
     # some entries in n.buses are not updated in previous functions, therefore can be wrong. as they are not needed
     # and are lost when clustering (for example with the simpl wildcard), we remove them for consistency:
     remove = [
@@ -621,6 +609,18 @@ if __name__ == "__main__":
     n.buses.drop(remove, axis=1, inplace=True, errors="ignore")
     n.lines.drop(remove, axis=1, errors="ignore", inplace=True)
 
+    if snakemake.wildcards.simpl:
+        n, cluster_map = cluster(
+            n,
+            int(snakemake.wildcards.simpl),
+            params.focus_weights,
+            solver_name,
+            params.simplify_network["algorithm"],
+            params.simplify_network["feature"],
+            params.aggregation_strategies,
+        )
+        busmaps.append(cluster_map)
+
     update_p_nom_max(n)
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))

scripts/solve_network.py

@@ -155,7 +155,7 @@ def _add_land_use_constraint(n):
 def _add_land_use_constraint_m(n, planning_horizons, config):
     # if generators clustering is lower than network clustering, land_use accounting is at generators clusters
 
-    grouping_years = config["existing_capacities"]["grouping_years"]
+    grouping_years = config["existing_capacities"]["grouping_years_power"]
     current_horizon = snakemake.wildcards.planning_horizons
 
     for carrier in ["solar", "onwind", "offwind-ac", "offwind-dc"]: