rearrange config settings under 'run: shared_resources:'

Snakefile

@@ -24,9 +24,11 @@ run = config["run"]
 scenarios = get_scenarios(run)
 RDIR = get_rdir(run)
 
-logs = path_provider("logs/", RDIR, run["shared_resources"], run["exclude"])
+shared_resources = run["shared_resources"]["mode"]
-benchmarks = path_provider("benchmarks/", RDIR, run["shared_resources"], run["exclude"])
+exclude_from_shared = run["shared_resources"]["exclude"]
-resources = path_provider("resources/", RDIR, run["shared_resources"], run["exclude"])
+logs = path_provider("logs/", RDIR, shared_resources, exclude_from_shared)
+benchmarks = path_provider("benchmarks/", RDIR, shared_resources, exclude_from_shared)
+resources = path_provider("resources/", RDIR, shared_resources, exclude_from_shared)
 
 CDIR = "" if run["shared_cutouts"] else RDIR
 RESULTS = "results/" + RDIR

config/config.default.yaml

@@ -26,9 +26,9 @@ run:
     enable: false
     file: config/scenarios.yaml
   disable_progressbar: false
-  shared_resources: false
+  shared_resources:
-  exclude:
+    policy: false
-  - ""
+    exclude: []
   shared_cutouts: true
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight

config/config.entsoe-all.yaml

@@ -5,7 +5,8 @@
 run:
   name: "entsoe-all"
   disable_progressbar: true
-  shared_resources: false
+  shared_resources:
+    policy: false
   shared_cutouts: true
 
 scenario:

config/test/config.electricity.yaml

@@ -8,7 +8,8 @@ tutorial: true
 run:
   name: "test-elec" # use this to keep track of runs with different settings
   disable_progressbar: true
-  shared_resources: "test"
+  shared_resources:
+    policy: "test"
   shared_cutouts: true
 
 scenario:

config/test/config.myopic.yaml

@@ -7,7 +7,8 @@ tutorial: true
 run:
   name: "test-sector-myopic"
   disable_progressbar: true
-  shared_resources: "test"
+  shared_resources:
+    policy: "test"
   shared_cutouts: true
 
 foresight: myopic

config/test/config.overnight.yaml

@@ -7,7 +7,8 @@ tutorial: true
 run:
   name: "test-sector-overnight"
   disable_progressbar: true
-  shared_resources: "test"
+  shared_resources:
+    policy: "test"
   shared_cutouts: true
 
 

config/test/config.perfect.yaml

@@ -7,7 +7,8 @@ tutorial: true
 run:
   name: "test-sector-perfect"
   disable_progressbar: true
-  shared_resources: "test"
+  shared_resources:
+    policy: "test"
   shared_cutouts: true
 
 foresight: perfect

config/test/config.scenarios.yaml

@@ -12,7 +12,8 @@ run:
     enable: true
     file: "config/test/scenarios.yaml"
   disable_progressbar: true
-  shared_resources: base
+  shared_resources:
+    policy: base
   shared_cutouts: true
 
 scenario:

doc/configtables/run.csv

@@ -5,6 +5,7 @@ scenarios,,,
 -- enable,bool,"{true, false}","Switch to select whether workflow should generate scenarios based on ``file``."
 -- file,str,,"Path to the scenario yaml file. The scenario file contains config overrides for each scenario. In order to be taken account, ``run: scenarios`` has to be set to ``true`` and ``run: name`` has to be a subset of top level keys given in the scenario file. In order to automatically create a `scenario.yaml` file based on a combination of settings, alter and use the ``config/create_scenarios.py`` script in the ``config`` directory."
 disable_progressbar,bool,"{true, false}","Switch to select whether progressbar should be disabled."
-shared_resources,bool/str,,"Switch to select whether resources should be shared across runs. If a string is passed, this is used as a subdirectory name for shared resources. If set to 'base', only resources before creating the elec.nc file are shared."
+shared_resources,,,
+-- policy,bool/str,,"Boolean switch to select whether resources should be shared across runs. If a string is passed, this is used as a subdirectory name for shared resources. If set to 'base', only resources before creating the elec.nc file are shared."
+-- exclude,str,"For the case shared_resources=base, specify additional files that should not be shared across runs."
 shared_cutouts,bool,"{true, false}","Switch to select whether cutouts should be shared across runs."
-exclude,str,"Specify files that should be excluded from shared_resources. Only active if shared_resources=base"

doc/release_notes.rst

@@ -12,7 +12,9 @@ Upcoming Release
 
 * Add floating wind technology for water depths below 60m
 
-* Add config [run][exclude] to specify which files should be excluded from filesharing in case of [run][shared_resources]=base.
+* Add config ``run: shared_resources: exclude:`` to specify additional files that should be excluded from shared resources with the setting ``run: shared_resources: base``.
+
+* Move switch ``run: shared_resources:`` to ``run: shared_resources: policy:``.
 
 * Add config land_transport_demand_factor to model growth in land transport demand for different time horizons.
 

scripts/_helpers.py

@@ -67,7 +67,7 @@ def get_rdir(run):
     return RDIR
 
 
-def get_run_path(fn, dir, rdir, shared_resources, exclude):
+def get_run_path(fn, dir, rdir, shared_resources, exclude_from_shared):
     """
     Dynamically provide paths based on shared resources and filename.
 
@@ -87,7 +87,7 @@ def get_run_path(fn, dir, rdir, shared_resources, exclude):
         - If string is "base", special handling for shared "base" resources (see notes).
         - If random string other than "base", this folder is used instead of the `rdir` keyword.
         - If boolean, directly specifies if the resource is shared.
-    exclude: list
+    exclude_from_shared: list
         List of filenames to exclude from shared resources. Only relevant if shared_resources is "base".
 
     Returns
@@ -106,12 +106,12 @@ def get_run_path(fn, dir, rdir, shared_resources, exclude):
         existing_wildcards = set(re.findall(pattern, fn))
         irrelevant_wildcards = {"technology", "year", "scope", "kind"}
         no_relevant_wildcards = not existing_wildcards - irrelevant_wildcards
-        no_elec_rule = (
+        not_shared_rule = (
             not fn.startswith("networks/elec")
             and not fn.startswith("add_electricity")
-            and not any(fn.startswith(ex) for ex in exclude)
+            and not any(fn.startswith(ex) for ex in exclude_from_shared)
         )
-        is_shared = no_relevant_wildcards and no_elec_rule
+        is_shared = no_relevant_wildcards and not_shared_rule
         rdir = "" if is_shared else rdir
     elif isinstance(shared_resources, str):
         rdir = shared_resources + "/"
@@ -125,7 +125,7 @@ def get_run_path(fn, dir, rdir, shared_resources, exclude):
     return f"{dir}{rdir}{fn}"
 
 
-def path_provider(dir, rdir, shared_resources, exclude):
+def path_provider(dir, rdir, shared_resources, exclude_from_shared):
     """
     Returns a partial function that dynamically provides paths based on shared
     resources and the filename.
@@ -141,7 +141,7 @@ def path_provider(dir, rdir, shared_resources, exclude):
         dir=dir,
         rdir=rdir,
         shared_resources=shared_resources,
-        exclude=exclude,
+        exclude_from_shared=exclude_from_shared,
     )