From 2565a7db4f283317a33e3a10c09b03463d2cb1fb Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 Mar 2024 08:20:23 +0100 Subject: [PATCH 1/4] snakefile: move copy_default_files and process_run_config to helpers --- Snakefile | 30 ++++++------------------------ scripts/_helpers.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/Snakefile b/Snakefile index 58cb6677..4a41622e 100644 --- a/Snakefile +++ b/Snakefile @@ -2,26 +2,18 @@ # # SPDX-License-Identifier: MIT -from os.path import normpath, exists -from shutil import copyfile, move, rmtree from pathlib import Path import yaml - +from os.path import normpath +from shutil import move, rmtree from snakemake.utils import min_version min_version("8.5") -from scripts._helpers import path_provider +from scripts._helpers import path_provider, copy_default_files, process_run_config -default_files = { - "config/config.default.yaml": "config/config.yaml", - "config/scenarios.template.yaml": "config/scenarios.yaml", -} -for template, target in default_files.items(): - target = os.path.join(workflow.current_basedir, target) - template = os.path.join(workflow.current_basedir, template) - if not exists(target) and exists(template): - copyfile(template, target) + +copy_default_files(workflow) configfile: "config/config.default.yaml" @@ -29,17 +21,7 @@ configfile: "config/config.yaml" run = config["run"] -scenarios = run.get("scenarios", {}) -if run["name"] and scenarios.get("enable"): - fn = Path(scenarios["file"]) - scenarios = yaml.safe_load(fn.read_text()) - RDIR = "{run}/" - if run["name"] == "all": - config["run"]["name"] = list(scenarios.keys()) -elif run["name"]: - RDIR = run["name"] + "/" -else: - RDIR = "" +RDIR = process_run_config(run) logs = path_provider("logs/", RDIR, run["shared_resources"]) benchmarks = path_provider("benchmarks/", RDIR, run["shared_resources"]) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index d03a306b..2bacf5cd 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -11,7 +11,9 @@ import os import re import urllib from functools import partial +from os.path import exists from pathlib import Path +from shutil import copyfile import pandas as pd import pytz @@ -25,6 +27,33 @@ logger = logging.getLogger(__name__) REGION_COLS = ["geometry", "name", "x", "y", "country"] +def copy_default_files(workflow): + default_files = { + "config/config.default.yaml": "config/config.yaml", + "config/scenarios.template.yaml": "config/scenarios.yaml", + } + for template, target in default_files.items(): + target = os.path.join(workflow.current_basedir, target) + template = os.path.join(workflow.current_basedir, template) + if not exists(target) and exists(template): + copyfile(template, target) + + +def process_run_config(run): + scenarios = run.get("scenarios", {}) + if run["name"] and scenarios.get("enable"): + fn = Path(scenarios["file"]) + scenarios = yaml.safe_load(fn.read_text()) + RDIR = "{run}/" + if run["name"] == "all": + run["name"] = list(scenarios.keys()) + elif run["name"]: + RDIR = run["name"] + "/" + else: + RDIR = "" + return RDIR + + def get_run_path(fn, dir, rdir, shared_resources): """ Dynamically provide paths based on shared resources and filename. From 23e1139c212aa449059ae82f975555142e20e5ec Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 Mar 2024 09:39:35 +0100 Subject: [PATCH 2/4] snakefile + helpers: separate scenario and rdir getter --- Snakefile | 5 +++-- scripts/_helpers.py | 17 ++++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/Snakefile b/Snakefile index 4a41622e..3e8875d3 100644 --- a/Snakefile +++ b/Snakefile @@ -10,7 +10,7 @@ from snakemake.utils import min_version min_version("8.5") -from scripts._helpers import path_provider, copy_default_files, process_run_config +from scripts._helpers import path_provider, copy_default_files, get_scenarios, get_rdir copy_default_files(workflow) @@ -21,7 +21,8 @@ configfile: "config/config.yaml" run = config["run"] -RDIR = process_run_config(run) +scenarios = get_scenarios(run) +RDIR = get_rdir(run) logs = path_provider("logs/", RDIR, run["shared_resources"]) benchmarks = path_provider("benchmarks/", RDIR, run["shared_resources"]) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 2bacf5cd..9b390380 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -39,14 +39,21 @@ def copy_default_files(workflow): copyfile(template, target) -def process_run_config(run): - scenarios = run.get("scenarios", {}) - if run["name"] and scenarios.get("enable"): - fn = Path(scenarios["file"]) +def get_scenarios(run): + scenario_config = run.get("scenarios", {}) + if run["name"] and scenario_config.get("enable"): + fn = Path(scenario_config["file"]) scenarios = yaml.safe_load(fn.read_text()) - RDIR = "{run}/" if run["name"] == "all": run["name"] = list(scenarios.keys()) + return scenarios + return {} + + +def get_rdir(run): + scenario_config = run.get("scenarios", {}) + if run["name"] and scenario_config.get("enable"): + RDIR = "{run}/" elif run["name"]: RDIR = run["name"] + "/" else: From 8190fde9cee654a8dafd1fd2df09c18e116e4372 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 Mar 2024 09:48:52 +0100 Subject: [PATCH 3/4] Snakefile: reinsert shutil and os functions --- Snakefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Snakefile b/Snakefile index 3e8875d3..4f55085f 100644 --- a/Snakefile +++ b/Snakefile @@ -4,8 +4,8 @@ from pathlib import Path import yaml -from os.path import normpath -from shutil import move, rmtree +from os.path import normpath, exists +from shutil import copyfile, move, rmtree from snakemake.utils import min_version min_version("8.5") From 86a6c3223def9ef304988eb9b0a53c973005bbf3 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 Mar 2024 11:26:36 +0100 Subject: [PATCH 4/4] perfect foresight: store configs in results dir --- rules/solve_perfect.smk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 215757fe..d4cbd6f3 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -110,7 +110,8 @@ rule solve_sector_network_perfect: output: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", - config="configs/config.elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.yaml", + config=RESULTS + + "configs/config.elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.yaml", threads: solver_threads resources: mem_mb=config_provider("solving", "mem"),