From 6321a698700f2e34fb471aaef57858846dbb57f6 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 13 May 2024 14:08:38 +0200 Subject: [PATCH] rearrange config settings under 'run: shared_resources:' --- Snakefile | 8 +++++--- config/config.default.yaml | 6 +++--- config/config.entsoe-all.yaml | 3 ++- config/test/config.electricity.yaml | 3 ++- config/test/config.myopic.yaml | 3 ++- config/test/config.overnight.yaml | 3 ++- config/test/config.perfect.yaml | 3 ++- config/test/config.scenarios.yaml | 3 ++- doc/configtables/run.csv | 5 +++-- doc/release_notes.rst | 4 +++- scripts/_helpers.py | 14 +++++++------- 11 files changed, 33 insertions(+), 22 deletions(-) diff --git a/Snakefile b/Snakefile index ce178619..9807c673 100644 --- a/Snakefile +++ b/Snakefile @@ -24,9 +24,11 @@ run = config["run"] scenarios = get_scenarios(run) RDIR = get_rdir(run) -logs = path_provider("logs/", RDIR, run["shared_resources"], run["exclude"]) -benchmarks = path_provider("benchmarks/", RDIR, run["shared_resources"], run["exclude"]) -resources = path_provider("resources/", RDIR, run["shared_resources"], run["exclude"]) +shared_resources = run["shared_resources"]["mode"] +exclude_from_shared = run["shared_resources"]["exclude"] +logs = path_provider("logs/", RDIR, shared_resources, exclude_from_shared) +benchmarks = path_provider("benchmarks/", RDIR, shared_resources, exclude_from_shared) +resources = path_provider("resources/", RDIR, shared_resources, exclude_from_shared) CDIR = "" if run["shared_cutouts"] else RDIR RESULTS = "results/" + RDIR diff --git a/config/config.default.yaml b/config/config.default.yaml index c75d39dd..92160d6a 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -26,9 +26,9 @@ run: enable: false file: config/scenarios.yaml disable_progressbar: false - shared_resources: false - exclude: - - "" + shared_resources: + policy: false + exclude: [] shared_cutouts: true # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight diff --git a/config/config.entsoe-all.yaml b/config/config.entsoe-all.yaml index 85eedb64..4e7edd03 100644 --- a/config/config.entsoe-all.yaml +++ b/config/config.entsoe-all.yaml @@ -5,7 +5,8 @@ run: name: "entsoe-all" disable_progressbar: true - shared_resources: false + shared_resources: + policy: false shared_cutouts: true scenario: diff --git a/config/test/config.electricity.yaml b/config/test/config.electricity.yaml index 979453cc..249a2609 100644 --- a/config/test/config.electricity.yaml +++ b/config/test/config.electricity.yaml @@ -8,7 +8,8 @@ tutorial: true run: name: "test-elec" # use this to keep track of runs with different settings disable_progressbar: true - shared_resources: "test" + shared_resources: + policy: "test" shared_cutouts: true scenario: diff --git a/config/test/config.myopic.yaml b/config/test/config.myopic.yaml index a9335046..a0d67940 100644 --- a/config/test/config.myopic.yaml +++ b/config/test/config.myopic.yaml @@ -7,7 +7,8 @@ tutorial: true run: name: "test-sector-myopic" disable_progressbar: true - shared_resources: "test" + shared_resources: + policy: "test" shared_cutouts: true foresight: myopic diff --git a/config/test/config.overnight.yaml b/config/test/config.overnight.yaml index b02c0449..56206fda 100644 --- a/config/test/config.overnight.yaml +++ b/config/test/config.overnight.yaml @@ -7,7 +7,8 @@ tutorial: true run: name: "test-sector-overnight" disable_progressbar: true - shared_resources: "test" + shared_resources: + policy: "test" shared_cutouts: true diff --git a/config/test/config.perfect.yaml b/config/test/config.perfect.yaml index 7fbfb630..eaabe6ef 100644 --- a/config/test/config.perfect.yaml +++ b/config/test/config.perfect.yaml @@ -7,7 +7,8 @@ tutorial: true run: name: "test-sector-perfect" disable_progressbar: true - shared_resources: "test" + shared_resources: + policy: "test" shared_cutouts: true foresight: perfect diff --git a/config/test/config.scenarios.yaml b/config/test/config.scenarios.yaml index 8ecbb91b..06fe9f23 100644 --- a/config/test/config.scenarios.yaml +++ b/config/test/config.scenarios.yaml @@ -12,7 +12,8 @@ run: enable: true file: "config/test/scenarios.yaml" disable_progressbar: true - shared_resources: base + shared_resources: + policy: base shared_cutouts: true scenario: diff --git a/doc/configtables/run.csv b/doc/configtables/run.csv index d938c04b..f11a8d96 100644 --- a/doc/configtables/run.csv +++ b/doc/configtables/run.csv @@ -5,6 +5,7 @@ scenarios,,, -- enable,bool,"{true, false}","Switch to select whether workflow should generate scenarios based on ``file``." -- file,str,,"Path to the scenario yaml file. The scenario file contains config overrides for each scenario. In order to be taken account, ``run: scenarios`` has to be set to ``true`` and ``run: name`` has to be a subset of top level keys given in the scenario file. In order to automatically create a `scenario.yaml` file based on a combination of settings, alter and use the ``config/create_scenarios.py`` script in the ``config`` directory." disable_progressbar,bool,"{true, false}","Switch to select whether progressbar should be disabled." -shared_resources,bool/str,,"Switch to select whether resources should be shared across runs. If a string is passed, this is used as a subdirectory name for shared resources. If set to 'base', only resources before creating the elec.nc file are shared." +shared_resources,,, +-- policy,bool/str,,"Boolean switch to select whether resources should be shared across runs. If a string is passed, this is used as a subdirectory name for shared resources. If set to 'base', only resources before creating the elec.nc file are shared." +-- exclude,str,"For the case shared_resources=base, specify additional files that should not be shared across runs." shared_cutouts,bool,"{true, false}","Switch to select whether cutouts should be shared across runs." -exclude,str,"Specify files that should be excluded from shared_resources. Only active if shared_resources=base" diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 517aa926..9d210a79 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -12,7 +12,9 @@ Upcoming Release * Add floating wind technology for water depths below 60m -* Add config [run][exclude] to specify which files should be excluded from filesharing in case of [run][shared_resources]=base. +* Add config ``run: shared_resources: exclude:`` to specify additional files that should be excluded from shared resources with the setting ``run: shared_resources: base``. + +* Move switch ``run: shared_resources:`` to ``run: shared_resources: policy:``. * Add config land_transport_demand_factor to model growth in land transport demand for different time horizons. diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 1301a296..5008b32a 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -67,7 +67,7 @@ def get_rdir(run): return RDIR -def get_run_path(fn, dir, rdir, shared_resources, exclude): +def get_run_path(fn, dir, rdir, shared_resources, exclude_from_shared): """ Dynamically provide paths based on shared resources and filename. @@ -87,7 +87,7 @@ def get_run_path(fn, dir, rdir, shared_resources, exclude): - If string is "base", special handling for shared "base" resources (see notes). - If random string other than "base", this folder is used instead of the `rdir` keyword. - If boolean, directly specifies if the resource is shared. - exclude: list + exclude_from_shared: list List of filenames to exclude from shared resources. Only relevant if shared_resources is "base". Returns @@ -106,12 +106,12 @@ def get_run_path(fn, dir, rdir, shared_resources, exclude): existing_wildcards = set(re.findall(pattern, fn)) irrelevant_wildcards = {"technology", "year", "scope", "kind"} no_relevant_wildcards = not existing_wildcards - irrelevant_wildcards - no_elec_rule = ( + not_shared_rule = ( not fn.startswith("networks/elec") and not fn.startswith("add_electricity") - and not any(fn.startswith(ex) for ex in exclude) + and not any(fn.startswith(ex) for ex in exclude_from_shared) ) - is_shared = no_relevant_wildcards and no_elec_rule + is_shared = no_relevant_wildcards and not_shared_rule rdir = "" if is_shared else rdir elif isinstance(shared_resources, str): rdir = shared_resources + "/" @@ -125,7 +125,7 @@ def get_run_path(fn, dir, rdir, shared_resources, exclude): return f"{dir}{rdir}{fn}" -def path_provider(dir, rdir, shared_resources, exclude): +def path_provider(dir, rdir, shared_resources, exclude_from_shared): """ Returns a partial function that dynamically provides paths based on shared resources and the filename. @@ -141,7 +141,7 @@ def path_provider(dir, rdir, shared_resources, exclude): dir=dir, rdir=rdir, shared_resources=shared_resources, - exclude=exclude, + exclude_from_shared=exclude_from_shared, )