further progress and bugfixes

.gitignore

@@ -27,7 +27,7 @@ doc/_build
 /scripts/create_scenarios.py
 /config/create_scenarios.py
 
-config.yaml
+config/config.yaml
 config/scenarios.yaml
 
 

Snakefile

@@ -20,9 +20,6 @@ configfile: "config/config.default.yaml"
 configfile: "config/config.yaml"
 
 
-COSTS = f"resources/costs_{config['costs']['year']}.csv"
-ATLITE_NPROCESSES = config["atlite"].get("nprocesses", 4)
-
 run = config["run"]
 scenarios = run.get("scenarios", {})
 if run["name"]:
@@ -39,16 +36,17 @@ benchmarks = path_provider("benchmarks/", RDIR, run["shared_resources"])
 resources = path_provider("resources/", RDIR, run["shared_resources"])
 
 CDIR = "" if run["shared_cutouts"] else RDIR
-LOGS = "logs/" + RDIR
-BENCHMARKS = "benchmarks/" + RDIR
-if not (shared_resources := run.get("shared_resources")):
-    RESOURCES = "resources/" + RDIR
-elif isinstance(shared_resources, str):
-    RESOURCES = "resources/" + shared_resources + "/"
-else:
-    RESOURCES = "resources/"
 RESULTS = "results/" + RDIR
 
+# TODO: this needs to be aligned with new scenario management
+# if not (shared_resources := run.get("shared_resources")):
+#     RESOURCES = "resources/" + RDIR
+# elif isinstance(shared_resources, str):
+#     RESOURCES = "resources/" + shared_resources + "/"
+# else:
+#     RESOURCES = "resources/"
+
+
 
 localrules:
     purge,

config/config.default.yaml

@@ -25,7 +25,7 @@ run:
     enable: false
     file: config/scenarios.yaml
   disable_progressbar: false
-  shared_resources: false
+  shared_resources: false # TODO: splitting resources by wildcard does not work well, neither does true, only base works well
   shared_cutouts: true
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight

rules/build_electricity.smk

@@ -162,12 +162,12 @@ if config["enable"].get("build_cutout", False):
         output:
             protected("cutouts/" + CDIR + "{cutout}.nc"),
         log:
-            "logs/" + CDIR + "build_cutout/{cutout}.log",
+            logs(CDIR + "build_cutout/{cutout}.log"),
         benchmark:
             "benchmarks/" + CDIR + "build_cutout_{cutout}"
-        threads: ATLITE_NPROCESSES
+        threads: config["atlite"].get("nprocesses", 4)
         resources:
-            mem_mb=ATLITE_NPROCESSES * 1000,
+            mem_mb=config["atlite"].get("nprocesses", 4) * 1000,
         conda:
             "../envs/environment.yaml"
         script:
@@ -249,9 +249,9 @@ rule determine_availability_matrix_MD_UA:
         availability_map=resources("availability_matrix_MD-UA_{technology}.png"),
     log:
         logs("determine_availability_matrix_MD_UA_{technology}.log"),
-    threads: ATLITE_NPROCESSES
+    threads: config["atlite"].get("nprocesses", 4)
     resources:
-        mem_mb=ATLITE_NPROCESSES * 5000,
+        mem_mb=config["atlite"].get("nprocesses", 4) * 5000,
     conda:
         "../envs/environment.yaml"
     script:
@@ -319,9 +319,9 @@ rule build_renewable_profiles:
         logs("build_renewable_profile_{technology}.log"),
     benchmark:
         benchmarks("build_renewable_profiles_{technology}")
-    threads: ATLITE_NPROCESSES
+    threads: config["atlite"].get("nprocesses", 4)
     resources:
-        mem_mb=ATLITE_NPROCESSES * 5000,
+        mem_mb=config["atlite"].get("nprocesses", 4) * 5000,
     wildcard_constraints:
         technology="(?!hydro).*",  # Any technology other than hydro
     conda:
@@ -391,9 +391,9 @@ if config["lines"]["dynamic_line_rating"]["activate"]:
             logs("build_line_rating.log"),
         benchmark:
             benchmarks("build_line_rating")
-        threads: ATLITE_NPROCESSES
+        threads: config["atlite"].get("nprocesses", 4)
         resources:
-            mem_mb=ATLITE_NPROCESSES * 1000,
+            mem_mb=config["atlite"].get("nprocesses", 4) * 1000,
         conda:
             "../envs/environment.yaml"
         script:
@@ -435,7 +435,7 @@ rule add_electricity:
             if config_provider("lines", "dynamic_line_rating", "activate")(w)
             else resources("networks/base.nc")
         ),
-        tech_costs=COSTS,
+        tech_costs=resources(f"costs_{config['costs']['year']}.csv"),
         regions=resources("regions_onshore.geojson"),
         powerplants=resources("powerplants.csv"),
         hydro_capacities=ancient("data/bundle/hydro_capacities.csv"),
@@ -478,7 +478,7 @@ rule simplify_network:
         costs=config_provider("costs"),
     input:
         network=resources("networks/elec.nc"),
-        tech_costs=COSTS,
+        tech_costs=resources(f"costs_{config['costs']['year']}.csv"),
         regions_onshore=resources("regions_onshore.geojson"),
         regions_offshore=resources("regions_offshore.geojson"),
     output:
@@ -525,7 +525,7 @@ rule cluster_network:
             if config_provider("enable", "custom_busmap", default=False)(w)
             else []
         ),
-        tech_costs=COSTS,
+        tech_costs=resources(f"costs_{config['costs']['year']}.csv"),
     output:
         network=resources("networks/elec_s{simpl}_{clusters}.nc"),
         regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
@@ -552,7 +552,7 @@ rule add_extra_components:
         costs=config_provider("costs"),
     input:
         network=resources("networks/elec_s{simpl}_{clusters}.nc"),
-        tech_costs=COSTS,
+        tech_costs=resources(f"costs_{config['costs']['year']}.csv"),
     output:
         resources("networks/elec_s{simpl}_{clusters}_ec.nc"),
     log:
@@ -570,11 +570,11 @@ rule add_extra_components:
 
 rule prepare_network:
     params:
-        snapshots={
+        snapshots=lambda w: {
-            "resolution": config_provider("snapshots", "resolution", default=False),
+            "resolution": config_provider("snapshots", "resolution", default=False)(w),
             "segmentation": config_provider(
                 "snapshots", "segmentation", default=False
-            ),
+            )(w),
         },
         links=config_provider("links"),
         lines=config_provider("lines"),
@@ -588,7 +588,7 @@ rule prepare_network:
         autarky=config_provider("electricity", "autarky", default={}),
     input:
         resources("networks/elec_s{simpl}_{clusters}_ec.nc"),
-        tech_costs=COSTS,
+        tech_costs=resources(f"costs_{config['costs']['year']}.csv"),
         co2_price=lambda w: resources("co2_price.csv") if "Ept" in w.opts else [],
     output:
         resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"),

rules/build_sector.smk

@@ -857,7 +857,7 @@ rule prepare_sector_network:
         biomass_potentials=lambda w: (
             resources(
                 "biomass_potentials_s{simpl}_{clusters}_"
-                + "{}.csv".format(config_provider("biomass", "year"))
+                + "{}.csv".format(config_provider("biomass", "year")(w))
             )
             if config_provider("foresight")(w) == "overnight"
             else resources(
@@ -865,9 +865,9 @@ rule prepare_sector_network:
             )
         ),
         costs=lambda w: (
-            "resources/costs_{}.csv".format(config_provider("costs", "year"))
+            resources("costs_{}.csv".format(config_provider("costs", "year")(w)))
             if config_provider("foresight")(w) == "overnight"
-            else "resources/costs_{planning_horizons}.csv"
+            else resources("costs_{planning_horizons}.csv")
         ),
         profile_offwind_ac=resources("profile_offwind-ac.nc"),
         profile_offwind_dc=resources("profile_offwind-dc.nc"),

rules/postprocess.smk

@@ -18,7 +18,7 @@ if config["foresight"] != "perfect":
                 "regions_onshore_elec_s{simpl}_{clusters}.geojson"
             ),
         output:
-            map=RESULTS + "maps/power-network-s{simpl}-{clusters}.pdf",
+            map=resources("maps/power-network-s{simpl}-{clusters}.pdf"),
         threads: 1
         resources:
             mem_mb=4000,
@@ -149,8 +149,6 @@ rule copy_config:
     threads: 1
     resources:
         mem_mb=1000,
-    benchmark:
-        benchmarks("copy_config")
     conda:
         "../envs/environment.yaml"
     script:
@@ -168,10 +166,6 @@ rule make_summary:
         scenario=config_provider("scenario"),
         RDIR=RDIR,
     input:
-        expand(
-            RESULTS + "maps/power-network-s{simpl}-{clusters}.pdf",
-            **config["scenario"],
-        ),
         networks=expand(
             RESULTS
             + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
@@ -179,15 +173,16 @@ rule make_summary:
             run=config["run"]["name"],
         ),
         costs=lambda w: (
-            "resources/costs_{}.csv".format(config_provider("costs", "year"))
+            resources("costs_{}.csv".format(config_provider("costs", "year")(w)))
             if config_provider("foresight")(w) == "overnight"
-            else "resources/costs_{}.csv".format(
+            else resources("costs_{}.csv".format(
                 config_provider("scenario", "planning_horizons", 0)
-            )
+            ))
         ),
         ac_plot=expand(
-            RESULTS + "maps/power-network-s{simpl}-{clusters}.pdf",
+            resources("maps/power-network-s{simpl}-{clusters}.pdf"),
             **config["scenario"],
+            run=config["run"]["name"],
         ),
         costs_plot=expand(
             RESULTS
@@ -235,9 +230,7 @@ rule make_summary:
     resources:
         mem_mb=10000,
     log:
-        logs("make_summary.log"),
+        RESULTS + "logs/make_summary.log",
-    benchmark:
-        benchmarks("make_summary")
     conda:
         "../envs/environment.yaml"
     script:
@@ -267,9 +260,7 @@ rule plot_summary:
     resources:
         mem_mb=10000,
     log:
-        logs("plot_summary.log"),
+        RESULTS + "logs/plot_summary.log",
-    benchmark:
-        benchmarks("plot_summary")
     conda:
         "../envs/environment.yaml"
     script:

rules/retrieve.smk

@@ -88,10 +88,12 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", T
         output:
             resources("costs_{year}.csv"),
         log:
-            "logs/retrieve_cost_data_{year}.log",
+            logs("retrieve_cost_data_{year}.log"),
         resources:
             mem_mb=1000,
         retries: 2
+        conda:
+            "../envs/retrieve.yaml"
         script:
             "../scripts/retrieve_cost_data.py"
 
@@ -191,6 +193,8 @@ if config["enable"]["retrieve"]:
         resources:
             mem_mb=5000,
         retries: 2
+        conda:
+            "../envs/retrieve.yaml"
         script:
             "../scripts/retrieve_electricity_demand.py"
 

rules/solve_electricity.smk

@@ -2,7 +2,6 @@
 #
 # SPDX-License-Identifier: MIT
 
-
 rule solve_network:
     params:
         solving=config_provider("solving"),

rules/solve_myopic.smk

@@ -16,9 +16,9 @@ rule add_existing_baseyear:
         busmap_s=resources("busmap_elec_s{simpl}.csv"),
         busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"),
         clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"),
-        costs=lambda w: "resources/costs_{}.csv".format(
+        costs=lambda w: resources("costs_{}.csv".format(
             config_provider("scenario", "planning_horizons", 0)(w)
-        ),
+        )),
         cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"),
         cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"),
         existing_heating_distribution=resources(
@@ -76,7 +76,7 @@ rule add_brownfield:
         network=RESULTS
         + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
         network_p=solved_previous_horizon,  #solved network at previous time step
-        costs="resources/costs_{planning_horizons}.csv",
+        costs=resources("costs_{planning_horizons}.csv"),
         cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"),
         cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"),
     output:
@@ -114,7 +114,7 @@ rule solve_sector_network_myopic:
     input:
         network=RESULTS
         + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-        costs="resources/costs_{planning_horizons}.csv",
+        costs=resources("costs_{planning_horizons}.csv"),
         config=RESULTS + "config.yaml",
     output:
         RESULTS

rules/solve_perfect.smk

@@ -14,9 +14,9 @@ rule add_existing_baseyear:
         busmap_s=resources("busmap_elec_s{simpl}.csv"),
         busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"),
         clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"),
-        costs="resources/costs_{}.csv".format(
+        costs=resources("costs_{}.csv".format(
             config_provider("scenario", "planning_horizons", 0)
-        ),
+        )),
         cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"),
         cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"),
         existing_heating_distribution=resources(
@@ -98,7 +98,7 @@ rule solve_sector_network_perfect:
     input:
         network=RESULTS
         + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
-        costs="resources/costs_2030.csv",
+        costs=resources("costs_2030.csv"),
         config=RESULTS + "config.yaml",
     output:
         RESULTS
@@ -140,7 +140,7 @@ def input_networks_make_summary_perfect(w):
 rule make_summary_perfect:
     input:
         unpack(input_networks_make_summary_perfect),
-        costs="resources/costs_2020.csv",
+        costs=resources("costs_2020.csv"),
     output:
         nodal_costs=RESULTS + "csvs/nodal_costs.csv",
         nodal_capacities=RESULTS + "csvs/nodal_capacities.csv",

scripts/_helpers.py

@@ -52,15 +52,16 @@ def get_run_path(fn, dir, rdir, shared_resources):
 
     Notes
     -----
-    Special case for "base" allows no wildcards other than
+    Special case for "base" allows no wildcards other than "technology", "year"
-    "technology" and excludes filenames starting with "networks/elec" or
+    and "scope" and excludes filenames starting with "networks/elec" or
     "add_electricity".
     """
     pattern = r"\{([^{}]+)\}"
-    existing_wildcards = list(re.findall(pattern, fn))
+    existing_wildcards = set(re.findall(pattern, fn))
     if shared_resources == "base":
         # special case for shared "base" resources
-        no_relevant_wildcards = not len(set(existing_wildcards) - {"technology"})
+        irrelevant_wildcards = {"technology", "year", "scope"}
+        no_relevant_wildcards = not len(existing_wildcards - irrelevant_wildcards)
         no_elec_rule = not fn.startswith("networks/elec") and not fn.startswith(
             "add_electricity"
         )
@@ -68,7 +69,7 @@ def get_run_path(fn, dir, rdir, shared_resources):
     elif isinstance(shared_resources, (str, list)):
         if isinstance(shared_resources, str):
             shared_resources = [shared_resources]
-        is_shared = set(existing_wildcards).issubset(shared_resources)
+        is_shared = (existing_wildcards).issubset(shared_resources)
     else:
         is_shared = shared_resources