Merge branch 'master' into fill-missing-domestic-aviation

.github/workflows/ci.yaml

@@ -31,7 +31,7 @@ jobs:
         os:
         - ubuntu-latest
         - macos-latest
-        - windows-latest
+#        - windows-latest
         inhouse:
         - stable
         - master

Snakefile

@@ -2,26 +2,18 @@
 #
 # SPDX-License-Identifier: MIT
 
-from os.path import normpath, exists
-from shutil import copyfile, move, rmtree
 from pathlib import Path
 import yaml
-
+from os.path import normpath, exists
+from shutil import copyfile, move, rmtree
 from snakemake.utils import min_version
 
 min_version("8.5")
 
-from scripts._helpers import path_provider
+from scripts._helpers import path_provider, copy_default_files, get_scenarios, get_rdir
 
-default_files = {
-    "config/config.default.yaml": "config/config.yaml",
-    "config/scenarios.template.yaml": "config/scenarios.yaml",
-}
-for template, target in default_files.items():
-    target = os.path.join(workflow.current_basedir, target)
-    template = os.path.join(workflow.current_basedir, template)
-    if not exists(target) and exists(template):
-        copyfile(template, target)
+
+copy_default_files(workflow)
 
 
 configfile: "config/config.default.yaml"
@@ -29,17 +21,8 @@ configfile: "config/config.yaml"
 
 
 run = config["run"]
-scenarios = run.get("scenarios", {})
-if run["name"] and scenarios.get("enable"):
-    fn = Path(scenarios["file"])
-    scenarios = yaml.safe_load(fn.read_text())
-    RDIR = "{run}/"
-    if run["name"] == "all":
-        config["run"]["name"] = list(scenarios.keys())
-elif run["name"]:
-    RDIR = run["name"] + "/"
-else:
-    RDIR = ""
+scenarios = get_scenarios(run)
+RDIR = get_rdir(run)
 
 logs = path_provider("logs/", RDIR, run["shared_resources"])
 benchmarks = path_provider("benchmarks/", RDIR, run["shared_resources"])
@@ -56,9 +39,9 @@ localrules:
 wildcard_constraints:
     simpl="[a-zA-Z0-9]*",
     clusters="[0-9]+(m|c)?|all",
-    ll="(v|c)([0-9\.]+|opt)",
-    opts="[-+a-zA-Z0-9\.]*",
-    sector_opts="[-+a-zA-Z0-9\.\s]*",
+    ll=r"(v|c)([0-9\.]+|opt)",
+    opts=r"[-+a-zA-Z0-9\.]*",
+    sector_opts=r"[-+a-zA-Z0-9\.\s]*",
 
 
 include: "rules/common.smk"
@@ -127,7 +110,7 @@ rule dag:
     conda:
         "envs/environment.yaml"
     shell:
-        """
+        r"""
         snakemake --rulegraph all | sed -n "/digraph/,\$p" > {output.dot}
         dot -Tpdf -o {output.pdf} {output.dot}
         dot -Tpng -o {output.png} {output.dot}

data/links_tyndp.csv

@@ -26,3 +26,16 @@ NordBalt,Klaipeda (LT),Nybro (SE),450,,700,built,,https://en.wikipedia.org/wiki/
 Estlink 1,Harku (EE),Espoo (FI),105,,350,built,,https://en.wikipedia.org/wiki/Estlink,24.560278,59.384722,24.551667,60.203889
 Greenlink,Waterford (IE),Pembroke (UK),,180,500,under construction,,https://tyndp2022-project-platform.azurewebsites.net/projectsheets/transmission/286,-6.987,52.260,-4.986,51.686
 Celtic Interconnector,Aghada (IE),La Martyre (FR),,572,700,under consideration,,https://tyndp2022-project-platform.azurewebsites.net/projectsheets/transmission/107,-8.16642,51.91413,-4.184,48.459
+GiLA,Bordeaux (FR),Nantes (FR),,312,640,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,-1.209,46.901,-0.576,44.960
+HG North Tyrrhenian Corridor,Milan (IT),Viterbo (IT),,500,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,9.409,45.553,12.015,42.244
+HG Adriatic Corridor,Ferrara (IT),Foggia (IT),,582,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,11.661,44.855,15.550,41.513
+SAPEI 2,Fioumesanto (IT),Montalto (IT),,390,1000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,8.283,40.790,11.602,42.331
+HG Ionian-Tyrrhenian Corridor,Rossano (IT),Latina (IT),,496,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,16.629,39.568,12.779,41.430
+HG Ionian-Tyrrhenian Corridor 2,Rossano (IT),Catania (IT),,330,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,16.629,39.568,15.049,37.408
+Germany-UK Hybrid Interconnector,Fetteresso (UK),Emden (DE),800,,2000,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,-2.383,56.991,7.207,53.376
+NU-Link Interconnector,Hornsea (UK),Moerdijk (NL),,460,1200,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,-0.261,53.655,4.586,51.661
+APOLLO-LINK,La Farga (ES),La Spezia (IT),,725,2091,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,2.883,42.062,9.884,44.107
+Baltic WindConnector (BWC),Lubmin (DE),Lihula (EE),,960,2000,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,13.686,54.139,23.818,58.675
+High-Voltage Direct Current Interconnector Project Romania-Hungary,Constanta (RO),Albertirsa (HU),,930,2500,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,28.588,44.201,19.584,47.224
+Rhine-Main-Link,Ovelgönne (DE),Marxheim (DE),,433,4000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,8.379,53.315,8.435,50.078
+Green Aegean Interconnector,Arachthos (GR),Ottenhofen (DE),,600,3000,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,20.967,39.185,11.868,48.207

doc/release_notes.rst

@@ -10,6 +10,8 @@ Release Notes
 Upcoming Release
 ================
 
+* Allow the use of more solvers in clustering (Xpress, COPT, Gurobi, CPLEX, SCIP, MOSEK).
+
 * Enhanced support for choosing different weather years
   (https://github.com/PyPSA/pypsa-eur/pull/204):
 
@@ -59,6 +61,9 @@ Upcoming Release
 * Removed rule ``copy_config``. Instead, a config file is created for each
   network output of the ``solve_*`` rules, with the same content as ``n.meta``.
 
+* Added new HVDC transmission projects from `TYNDP 2024 draft projects
+  <https://tyndp.entsoe.eu/news/176-pan-european-electricity-transmission-projects-and-33-storage-projects-will-be-assessed-in-tyndp-2024>`_.
+
 * Upgrade to Snakemake v8.5+. This version is the new minimum version required.
   To upgrade an existing environment, run ``conda install -c bioconda
   snakemake-minimal">=8.5"`` and ``pip install snakemake-storage-plugin-http``

rules/retrieve.smk

@@ -115,7 +115,7 @@ if config["enable"]["retrieve"] and config["enable"].get(
             mem_mb=5000,
         retries: 2
         run:
-            move(input[0], output[0])
+            copyfile(input[0], output[0])
             validate_checksum(output[0], input[0])
 
 

rules/solve_electricity.smk

@@ -31,7 +31,7 @@ rule solve_network:
         mem_mb=memory,
         runtime=config_provider("solving", "runtime", default="6h"),
     shadow:
-        "minimal"
+        "shallow"
     conda:
         "../envs/environment.yaml"
     script:
@@ -62,7 +62,7 @@ rule solve_operations_network:
         mem_mb=(lambda w: 10000 + 372 * int(w.clusters)),
         runtime=config_provider("solving", "runtime", default="6h"),
     shadow:
-        "minimal"
+        "shallow"
     conda:
         "../envs/environment.yaml"
     script:

rules/solve_perfect.smk

@@ -110,7 +110,8 @@ rule solve_sector_network_perfect:
     output:
         network=RESULTS
         + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
-        config="configs/config.elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.yaml",
+        config=RESULTS
+        + "configs/config.elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.yaml",
     threads: solver_threads
     resources:
         mem_mb=config_provider("solving", "mem"),

scripts/_helpers.py

@@ -11,7 +11,9 @@ import os
 import re
 import urllib
 from functools import partial
+from os.path import exists
 from pathlib import Path
+from shutil import copyfile
 
 import pandas as pd
 import pytz
@@ -25,6 +27,40 @@ logger = logging.getLogger(__name__)
 REGION_COLS = ["geometry", "name", "x", "y", "country"]
 
 
+def copy_default_files(workflow):
+    default_files = {
+        "config/config.default.yaml": "config/config.yaml",
+        "config/scenarios.template.yaml": "config/scenarios.yaml",
+    }
+    for template, target in default_files.items():
+        target = os.path.join(workflow.current_basedir, target)
+        template = os.path.join(workflow.current_basedir, template)
+        if not exists(target) and exists(template):
+            copyfile(template, target)
+
+
+def get_scenarios(run):
+    scenario_config = run.get("scenarios", {})
+    if run["name"] and scenario_config.get("enable"):
+        fn = Path(scenario_config["file"])
+        scenarios = yaml.safe_load(fn.read_text())
+        if run["name"] == "all":
+            run["name"] = list(scenarios.keys())
+        return scenarios
+    return {}
+
+
+def get_rdir(run):
+    scenario_config = run.get("scenarios", {})
+    if run["name"] and scenario_config.get("enable"):
+        RDIR = "{run}/"
+    elif run["name"]:
+        RDIR = run["name"] + "/"
+    else:
+        RDIR = ""
+    return RDIR
+
+
 def get_run_path(fn, dir, rdir, shared_resources):
     """
     Dynamically provide paths based on shared resources and filename.
@@ -66,20 +102,17 @@ def get_run_path(fn, dir, rdir, shared_resources):
             "add_electricity"
         )
         is_shared = no_relevant_wildcards and no_elec_rule
+        rdir = "" if is_shared else rdir
     elif isinstance(shared_resources, str):
         rdir = shared_resources + "/"
-        is_shared = False
     elif isinstance(shared_resources, bool):
-        is_shared = shared_resources
+        rdir = ""
     else:
         raise ValueError(
             "shared_resources must be a boolean, str, or 'base' for special handling."
         )
 
-    if is_shared:
-        return f"{dir}{fn}"
-    else:
-        return f"{dir}{rdir}{fn}"
+    return f"{dir}{rdir}{fn}"
 
 
 def path_provider(dir, rdir, shared_resources):

scripts/add_brownfield.py

@@ -145,7 +145,7 @@ def disable_grid_expansion_if_limit_hit(n):
         for name, glc in glcs.iterrows():
             total_expansion = (
                 (
-                    n.lines.query("p_nom_extendable")
+                    n.lines.query("s_nom_extendable")
                     .eval(f"s_nom_min * {cols[limit_type]}")
                     .sum()
                 )

scripts/build_hydro_profile.py

@@ -202,9 +202,10 @@ if __name__ == "__main__":
 
     contained_years = pd.date_range(freq="YE", **snakemake.params.snapshots).year
     norm_year = config_hydro.get("eia_norm_year")
+    missing_years = contained_years.difference(eia_stats.index)
     if norm_year:
         eia_stats.loc[contained_years] = eia_stats.loc[norm_year]
-    elif missing_years := eia_stats.index.difference(contained_years):
+    elif missing_years.any():
         eia_stats.loc[missing_years] = eia_stats.median()
 
     inflow = cutout.runoff(

scripts/build_sequestration_potentials.py

@@ -44,7 +44,7 @@ if __name__ == "__main__":
 
     cf = snakemake.params.sequestration_potential
 
-    gdf = gpd.read_file(snakemake.input.sequestration_potential[0])
+    gdf = gpd.read_file(snakemake.input.sequestration_potential)
 
     regions = gpd.read_file(snakemake.input.regions_offshore)
     if cf["include_onshore"]:

scripts/cluster_network.py

@@ -267,7 +267,7 @@ def distribute_clusters(n, n_clusters, focus_weights=None, solver_name="scip"):
     m.objective = (clusters * clusters - 2 * clusters * L * n_clusters).sum()
     if solver_name == "gurobi":
         logging.getLogger("gurobipy").propagate = False
-    elif solver_name not in ["scip", "cplex"]:
+    elif solver_name not in ["scip", "cplex", "xpress", "copt", "mosek"]:
         logger.info(
             f"The configured solver `{solver_name}` does not support quadratic objectives. Falling back to `scip`."
         )

scripts/retrieve_cost_data.py

@@ -26,7 +26,7 @@ if __name__ == "__main__":
 
     version = snakemake.params.version
     if "/" in version:
-        baseurl = f"https://raw.githubusercontent.com/{version}/outputs"
+        baseurl = f"https://raw.githubusercontent.com/{version}/outputs/"
     else:
         baseurl = f"https://raw.githubusercontent.com/PyPSA/technology-data/{version}/outputs/"
     filepath = Path(snakemake.output[0])