From f3eae37a02ebf974228279777082d180f97c4a8a Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 14 Mar 2024 12:52:25 +0100 Subject: [PATCH 01/14] scneario management: reenable shared resources in one folder --- scripts/_helpers.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index d03a306b..5884d908 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -66,20 +66,17 @@ def get_run_path(fn, dir, rdir, shared_resources): "add_electricity" ) is_shared = no_relevant_wildcards and no_elec_rule + rdir = "" if is_shared else rdir elif isinstance(shared_resources, str): rdir = shared_resources + "/" - is_shared = False elif isinstance(shared_resources, bool): - is_shared = shared_resources + rdir = "" else: raise ValueError( "shared_resources must be a boolean, str, or 'base' for special handling." ) - if is_shared: - return f"{dir}{fn}" - else: - return f"{dir}{rdir}{fn}" + return f"{dir}{rdir}{fn}" def path_provider(dir, rdir, shared_resources): From 8d9afb0eb048f6d0d3c4370c84c29159862b4bec Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Tue, 19 Mar 2024 08:28:27 +0100 Subject: [PATCH 02/14] solve_electricity.smk:replace "minimal" by "shallow" --- rules/solve_electricity.smk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index f24119b9..389687a0 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -31,7 +31,7 @@ rule solve_network: mem_mb=memory, runtime=config_provider("solving", "runtime", default="6h"), shadow: - "minimal" + "shallow" conda: "../envs/environment.yaml" script: @@ -62,7 +62,7 @@ rule solve_operations_network: mem_mb=(lambda w: 10000 + 372 * int(w.clusters)), runtime=config_provider("solving", "runtime", default="6h"), shadow: - "minimal" + "shallow" conda: "../envs/environment.yaml" script: From 2565a7db4f283317a33e3a10c09b03463d2cb1fb Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 Mar 2024 08:20:23 +0100 Subject: [PATCH 03/14] snakefile: move copy_default_files and process_run_config to helpers --- Snakefile | 30 ++++++------------------------ scripts/_helpers.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/Snakefile b/Snakefile index 58cb6677..4a41622e 100644 --- a/Snakefile +++ b/Snakefile @@ -2,26 +2,18 @@ # # SPDX-License-Identifier: MIT -from os.path import normpath, exists -from shutil import copyfile, move, rmtree from pathlib import Path import yaml - +from os.path import normpath +from shutil import move, rmtree from snakemake.utils import min_version min_version("8.5") -from scripts._helpers import path_provider +from scripts._helpers import path_provider, copy_default_files, process_run_config -default_files = { - "config/config.default.yaml": "config/config.yaml", - "config/scenarios.template.yaml": "config/scenarios.yaml", -} -for template, target in default_files.items(): - target = os.path.join(workflow.current_basedir, target) - template = os.path.join(workflow.current_basedir, template) - if not exists(target) and exists(template): - copyfile(template, target) + +copy_default_files(workflow) configfile: "config/config.default.yaml" @@ -29,17 +21,7 @@ configfile: "config/config.yaml" run = config["run"] -scenarios = run.get("scenarios", {}) -if run["name"] and scenarios.get("enable"): - fn = Path(scenarios["file"]) - scenarios = yaml.safe_load(fn.read_text()) - RDIR = "{run}/" - if run["name"] == "all": - config["run"]["name"] = list(scenarios.keys()) -elif run["name"]: - RDIR = run["name"] + "/" -else: - RDIR = "" +RDIR = process_run_config(run) logs = path_provider("logs/", RDIR, run["shared_resources"]) benchmarks = path_provider("benchmarks/", RDIR, run["shared_resources"]) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index d03a306b..2bacf5cd 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -11,7 +11,9 @@ import os import re import urllib from functools import partial +from os.path import exists from pathlib import Path +from shutil import copyfile import pandas as pd import pytz @@ -25,6 +27,33 @@ logger = logging.getLogger(__name__) REGION_COLS = ["geometry", "name", "x", "y", "country"] +def copy_default_files(workflow): + default_files = { + "config/config.default.yaml": "config/config.yaml", + "config/scenarios.template.yaml": "config/scenarios.yaml", + } + for template, target in default_files.items(): + target = os.path.join(workflow.current_basedir, target) + template = os.path.join(workflow.current_basedir, template) + if not exists(target) and exists(template): + copyfile(template, target) + + +def process_run_config(run): + scenarios = run.get("scenarios", {}) + if run["name"] and scenarios.get("enable"): + fn = Path(scenarios["file"]) + scenarios = yaml.safe_load(fn.read_text()) + RDIR = "{run}/" + if run["name"] == "all": + run["name"] = list(scenarios.keys()) + elif run["name"]: + RDIR = run["name"] + "/" + else: + RDIR = "" + return RDIR + + def get_run_path(fn, dir, rdir, shared_resources): """ Dynamically provide paths based on shared resources and filename. From 65af55c22b48e305a6baa30f265054429ba79a55 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Tue, 19 Mar 2024 08:54:58 +0100 Subject: [PATCH 04/14] Fix typos --- scripts/add_brownfield.py | 2 +- scripts/retrieve_cost_data.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 82a40e57..7e49031b 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -145,7 +145,7 @@ def disable_grid_expansion_if_limit_hit(n): for name, glc in glcs.iterrows(): total_expansion = ( ( - n.lines.query("p_nom_extendable") + n.lines.query("s_nom_extendable") .eval(f"s_nom_min * {cols[limit_type]}") .sum() ) diff --git a/scripts/retrieve_cost_data.py b/scripts/retrieve_cost_data.py index 39fcab7c..e236fbfd 100644 --- a/scripts/retrieve_cost_data.py +++ b/scripts/retrieve_cost_data.py @@ -26,7 +26,7 @@ if __name__ == "__main__": version = snakemake.params.version if "/" in version: - baseurl = f"https://raw.githubusercontent.com/{version}/outputs" + baseurl = f"https://raw.githubusercontent.com/{version}/outputs/" else: baseurl = f"https://raw.githubusercontent.com/PyPSA/technology-data/{version}/outputs/" filepath = Path(snakemake.output[0]) From 22d1bf5dd1a35554d476263c91d9bb390881dc2f Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Mon, 18 Mar 2024 11:58:39 +0100 Subject: [PATCH 05/14] Bugfix for missing eia years First, cannot cast pd index to boolean. Second, fix logic error of mistakenly swapped difference in calculating missing years. --- scripts/build_hydro_profile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index b7f270b3..c4e9701e 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -202,9 +202,10 @@ if __name__ == "__main__": contained_years = pd.date_range(freq="YE", **snakemake.params.snapshots).year norm_year = config_hydro.get("eia_norm_year") + missing_years = contained_years.difference(eia_stats.index) if norm_year: eia_stats.loc[contained_years] = eia_stats.loc[norm_year] - elif missing_years := eia_stats.index.difference(contained_years): + elif missing_years.any(): eia_stats.loc[missing_years] = eia_stats.median() inflow = cutout.runoff( From fcf773c003254975740340bb8cb21827aef7fc8b Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Tue, 19 Mar 2024 08:28:14 +0000 Subject: [PATCH 06/14] Copy, don't move natura.tiff to resources Preventing problems when parallel processes attempt to move the same file to different resources directories. --- rules/retrieve.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index ac296f79..4b244483 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -115,7 +115,7 @@ if config["enable"]["retrieve"] and config["enable"].get( mem_mb=5000, retries: 2 run: - move(input[0], output[0]) + copyfile(input[0], output[0]) validate_checksum(output[0], input[0]) From 23e1139c212aa449059ae82f975555142e20e5ec Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 Mar 2024 09:39:35 +0100 Subject: [PATCH 07/14] snakefile + helpers: separate scenario and rdir getter --- Snakefile | 5 +++-- scripts/_helpers.py | 17 ++++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/Snakefile b/Snakefile index 4a41622e..3e8875d3 100644 --- a/Snakefile +++ b/Snakefile @@ -10,7 +10,7 @@ from snakemake.utils import min_version min_version("8.5") -from scripts._helpers import path_provider, copy_default_files, process_run_config +from scripts._helpers import path_provider, copy_default_files, get_scenarios, get_rdir copy_default_files(workflow) @@ -21,7 +21,8 @@ configfile: "config/config.yaml" run = config["run"] -RDIR = process_run_config(run) +scenarios = get_scenarios(run) +RDIR = get_rdir(run) logs = path_provider("logs/", RDIR, run["shared_resources"]) benchmarks = path_provider("benchmarks/", RDIR, run["shared_resources"]) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 2bacf5cd..9b390380 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -39,14 +39,21 @@ def copy_default_files(workflow): copyfile(template, target) -def process_run_config(run): - scenarios = run.get("scenarios", {}) - if run["name"] and scenarios.get("enable"): - fn = Path(scenarios["file"]) +def get_scenarios(run): + scenario_config = run.get("scenarios", {}) + if run["name"] and scenario_config.get("enable"): + fn = Path(scenario_config["file"]) scenarios = yaml.safe_load(fn.read_text()) - RDIR = "{run}/" if run["name"] == "all": run["name"] = list(scenarios.keys()) + return scenarios + return {} + + +def get_rdir(run): + scenario_config = run.get("scenarios", {}) + if run["name"] and scenario_config.get("enable"): + RDIR = "{run}/" elif run["name"]: RDIR = run["name"] + "/" else: From 8190fde9cee654a8dafd1fd2df09c18e116e4372 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 Mar 2024 09:48:52 +0100 Subject: [PATCH 08/14] Snakefile: reinsert shutil and os functions --- Snakefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Snakefile b/Snakefile index 3e8875d3..4f55085f 100644 --- a/Snakefile +++ b/Snakefile @@ -4,8 +4,8 @@ from pathlib import Path import yaml -from os.path import normpath -from shutil import move, rmtree +from os.path import normpath, exists +from shutil import copyfile, move, rmtree from snakemake.utils import min_version min_version("8.5") From 86a6c3223def9ef304988eb9b0a53c973005bbf3 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 Mar 2024 11:26:36 +0100 Subject: [PATCH 09/14] perfect foresight: store configs in results dir --- rules/solve_perfect.smk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 215757fe..d4cbd6f3 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -110,7 +110,8 @@ rule solve_sector_network_perfect: output: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", - config="configs/config.elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.yaml", + config=RESULTS + + "configs/config.elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.yaml", threads: solver_threads resources: mem_mb=config_provider("solving", "mem"), From 651e28775ecbe50f9918ea18bbb06edc57d58ad9 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Tue, 19 Mar 2024 13:00:16 +0000 Subject: [PATCH 10/14] Use raw strings to avoid illegal backslash warnings python 12 --- Snakefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Snakefile b/Snakefile index 58cb6677..ca1b72cc 100644 --- a/Snakefile +++ b/Snakefile @@ -56,9 +56,9 @@ localrules: wildcard_constraints: simpl="[a-zA-Z0-9]*", clusters="[0-9]+(m|c)?|all", - ll="(v|c)([0-9\.]+|opt)", - opts="[-+a-zA-Z0-9\.]*", - sector_opts="[-+a-zA-Z0-9\.\s]*", + ll=r"(v|c)([0-9\.]+|opt)", + opts=r"[-+a-zA-Z0-9\.]*", + sector_opts=r"[-+a-zA-Z0-9\.\s]*", include: "rules/common.smk" @@ -127,7 +127,7 @@ rule dag: conda: "envs/environment.yaml" shell: - """ + r""" snakemake --rulegraph all | sed -n "/digraph/,\$p" > {output.dot} dot -Tpdf -o {output.pdf} {output.dot} dot -Tpng -o {output.png} {output.dot} From c5026aaf9b24a4ec17049e03cf2f6953b7059f02 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 20 Mar 2024 13:56:48 +0100 Subject: [PATCH 11/14] add draft HVDC projects from TYNDP 2024 --- data/links_tyndp.csv | 13 +++++++++++++ doc/release_notes.rst | 3 +++ 2 files changed, 16 insertions(+) diff --git a/data/links_tyndp.csv b/data/links_tyndp.csv index a0603120..43030be5 100644 --- a/data/links_tyndp.csv +++ b/data/links_tyndp.csv @@ -26,3 +26,16 @@ NordBalt,Klaipeda (LT),Nybro (SE),450,,700,built,,https://en.wikipedia.org/wiki/ Estlink 1,Harku (EE),Espoo (FI),105,,350,built,,https://en.wikipedia.org/wiki/Estlink,24.560278,59.384722,24.551667,60.203889 Greenlink,Waterford (IE),Pembroke (UK),,180,500,under construction,,https://tyndp2022-project-platform.azurewebsites.net/projectsheets/transmission/286,-6.987,52.260,-4.986,51.686 Celtic Interconnector,Aghada (IE),La Martyre (FR),,572,700,under consideration,,https://tyndp2022-project-platform.azurewebsites.net/projectsheets/transmission/107,-8.16642,51.91413,-4.184,48.459 +GiLA,Bordeaux (FR),Nantes (FR),,312,640,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,-1.209,46.901,-0.576,44.960 +HG North Tyrrhenian Corridor,Milan (IT),Viterbo (IT),,500,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,9.409,45.553,12.015,42.244 +HG Adriatic Corridor,Ferrara (IT),Foggia (IT),,582,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,11.661,44.855,15.550,41.513 +SAPEI 2,Fioumesanto (IT),Montalto (IT),,390,1000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,8.283,40.790,11.602,42.331 +HG Ionian-Tyrrhenian Corridor,Rossano (IT),Latina (IT),,496,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,16.629,39.568,12.779,41.430 +HG Ionian-Tyrrhenian Corridor 2,Rossano (IT),Catania (IT),,330,2000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,16.629,39.568,15.049,37.408 +Germany-UK Hybrid Interconnector,Fetteresso (UK),Emden (DE),800,,2000,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,-2.383,56.991,7.207,53.376 +NU-Link Interconnector,Hornsea (UK),Moerdijk (NL),,460,1200,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,-0.261,53.655,4.586,51.661 +APOLLO-LINK,La Farga (ES),La Spezia (IT),,725,2091,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,2.883,42.062,9.884,44.107 +Baltic WindConnector (BWC),Lubmin (DE),Lihula (EE),,960,2000,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,13.686,54.139,23.818,58.675 +High-Voltage Direct Current Interconnector Project Romania-Hungary,Constanta (RO),Albertirsa (HU),,930,2500,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,28.588,44.201,19.584,47.224 +Rhine-Main-Link,Ovelgönne (DE),Marxheim (DE),,433,4000,in permitting,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,8.379,53.315,8.435,50.078 +Green Aegean Interconnector,Arachthos (GR),Ottenhofen (DE),,600,3000,under consideration,,https://eepublicdownloads.blob.core.windows.net/public-cdn-container/tyndp-documents/TYNDP2024/240220_TYNDP2024_project_portfolio.xlsx,20.967,39.185,11.868,48.207 diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 21094250..e081ce35 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -59,6 +59,9 @@ Upcoming Release * Removed rule ``copy_config``. Instead, a config file is created for each network output of the ``solve_*`` rules, with the same content as ``n.meta``. +* Added new HVDC transmission projects from `TYNDP 2024 draft projects + `_. + * Upgrade to Snakemake v8.5+. This version is the new minimum version required. To upgrade an existing environment, run ``conda install -c bioconda snakemake-minimal">=8.5"`` and ``pip install snakemake-storage-plugin-http`` From bb898bf69519c1b3c6454ff3248f2d42df543c2d Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Wed, 20 Mar 2024 15:40:07 +0100 Subject: [PATCH 12/14] Fix typo in reading input to build_sequestration_potentials rule Presumably needed since the transition to snakemake 8 storage providers --- scripts/build_sequestration_potentials.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py index 106c1271..2a362a77 100644 --- a/scripts/build_sequestration_potentials.py +++ b/scripts/build_sequestration_potentials.py @@ -44,7 +44,7 @@ if __name__ == "__main__": cf = snakemake.params.sequestration_potential - gdf = gpd.read_file(snakemake.input.sequestration_potential[0]) + gdf = gpd.read_file(snakemake.input.sequestration_potential) regions = gpd.read_file(snakemake.input.regions_offshore) if cf["include_onshore"]: From ab8336d6a891c89b2154a540ab10edcb61c73a06 Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Thu, 21 Mar 2024 09:55:48 +0100 Subject: [PATCH 13/14] disable windows machines in CI The CI on windows machines keeps failing due to some weird syntax problem in snakefiles, which we can't solve on our end it seems. --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c17c0425..7aabf0e6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -31,7 +31,7 @@ jobs: os: - ubuntu-latest - macos-latest - - windows-latest +# - windows-latest inhouse: - stable - master From d0951abdedd11c8ae4c044905673d270e56dc52e Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 21 Mar 2024 12:02:02 +0100 Subject: [PATCH 14/14] cluster_network: allow more solvers that can handle quadratic problems --- doc/release_notes.rst | 2 ++ scripts/cluster_network.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 21094250..2982f0a6 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,8 @@ Release Notes Upcoming Release ================ +* Allow the use of more solvers in clustering (Xpress, COPT, Gurobi, CPLEX, SCIP, MOSEK). + * Enhanced support for choosing different weather years (https://github.com/PyPSA/pypsa-eur/pull/204): diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 270774af..3f6c9ea2 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -267,7 +267,7 @@ def distribute_clusters(n, n_clusters, focus_weights=None, solver_name="scip"): m.objective = (clusters * clusters - 2 * clusters * L * n_clusters).sum() if solver_name == "gurobi": logging.getLogger("gurobipy").propagate = False - elif solver_name not in ["scip", "cplex"]: + elif solver_name not in ["scip", "cplex", "xpress", "copt", "mosek"]: logger.info( f"The configured solver `{solver_name}` does not support quadratic objectives. Falling back to `scip`." )