further progress and bugfixes

2024-02-16 11:17:00 +01:00 · 2024-02-16 11:17:00 +01:00 · 16e42c8fe6
commit 16e42c8fe6
parent dbbfbaf1f3
11 changed files with 58 additions and 65 deletions
--- a/.gitignore
+++ b/.gitignore
@ -27,7 +27,7 @@ doc/_build
 /scripts/create_scenarios.py
 /config/create_scenarios.py

-config.yaml
+config/config.yaml
 config/scenarios.yaml


--- a/20
+++ b/20
@ -20,9 +20,6 @@ configfile: "config/config.default.yaml"
 configfile: "config/config.yaml"


-COSTS = f"resources/costs_{config['costs']['year']}.csv"
-ATLITE_NPROCESSES = config["atlite"].get("nprocesses", 4)
-
 run = config["run"]
 scenarios = run.get("scenarios", {})
 if run["name"]:
@ -39,16 +36,17 @@ benchmarks = path_provider("benchmarks/", RDIR, run["shared_resources"])
 resources = path_provider("resources/", RDIR, run["shared_resources"])

 CDIR = "" if run["shared_cutouts"] else RDIR
-LOGS = "logs/" + RDIR
-BENCHMARKS = "benchmarks/" + RDIR
-if not (shared_resources := run.get("shared_resources")):
-    RESOURCES = "resources/" + RDIR
-elif isinstance(shared_resources, str):
-    RESOURCES = "resources/" + shared_resources + "/"
-else:
-    RESOURCES = "resources/"
 RESULTS = "results/" + RDIR

+# TODO: this needs to be aligned with new scenario management
+# if not (shared_resources := run.get("shared_resources")):
+#     RESOURCES = "resources/" + RDIR
+# elif isinstance(shared_resources, str):
+#     RESOURCES = "resources/" + shared_resources + "/"
+# else:
+#     RESOURCES = "resources/"
+
+

 localrules:
    purge,
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@ -25,7 +25,7 @@ run:
    enable: false
    file: config/scenarios.yaml
  disable_progressbar: false
-  shared_resources: false
+  shared_resources: false # TODO: splitting resources by wildcard does not work well, neither does true, only base works well
  shared_cutouts: true

 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@ -162,12 +162,12 @@ if config["enable"].get("build_cutout", False):
        output:
            protected("cutouts/" + CDIR + "{cutout}.nc"),
        log:
-            "logs/" + CDIR + "build_cutout/{cutout}.log",
+            logs(CDIR + "build_cutout/{cutout}.log"),
        benchmark:
            "benchmarks/" + CDIR + "build_cutout_{cutout}"
-        threads: ATLITE_NPROCESSES
+        threads: config["atlite"].get("nprocesses", 4)
        resources:
-            mem_mb=ATLITE_NPROCESSES * 1000,
+            mem_mb=config["atlite"].get("nprocesses", 4) * 1000,
        conda:
            "../envs/environment.yaml"
        script:
@ -249,9 +249,9 @@ rule determine_availability_matrix_MD_UA:
        availability_map=resources("availability_matrix_MD-UA_{technology}.png"),
    log:
        logs("determine_availability_matrix_MD_UA_{technology}.log"),
-    threads: ATLITE_NPROCESSES
+    threads: config["atlite"].get("nprocesses", 4)
    resources:
-        mem_mb=ATLITE_NPROCESSES * 5000,
+        mem_mb=config["atlite"].get("nprocesses", 4) * 5000,
    conda:
        "../envs/environment.yaml"
    script:
@ -319,9 +319,9 @@ rule build_renewable_profiles:
        logs("build_renewable_profile_{technology}.log"),
    benchmark:
        benchmarks("build_renewable_profiles_{technology}")
-    threads: ATLITE_NPROCESSES
+    threads: config["atlite"].get("nprocesses", 4)
    resources:
-        mem_mb=ATLITE_NPROCESSES * 5000,
+        mem_mb=config["atlite"].get("nprocesses", 4) * 5000,
    wildcard_constraints:
        technology="(?!hydro).*",  # Any technology other than hydro
    conda:
@ -391,9 +391,9 @@ if config["lines"]["dynamic_line_rating"]["activate"]:
            logs("build_line_rating.log"),
        benchmark:
            benchmarks("build_line_rating")
-        threads: ATLITE_NPROCESSES
+        threads: config["atlite"].get("nprocesses", 4)
        resources:
-            mem_mb=ATLITE_NPROCESSES * 1000,
+            mem_mb=config["atlite"].get("nprocesses", 4) * 1000,
        conda:
            "../envs/environment.yaml"
        script:
@ -435,7 +435,7 @@ rule add_electricity:
            if config_provider("lines", "dynamic_line_rating", "activate")(w)
            else resources("networks/base.nc")
        ),
-        tech_costs=COSTS,
+        tech_costs=resources(f"costs_{config['costs']['year']}.csv"),
        regions=resources("regions_onshore.geojson"),
        powerplants=resources("powerplants.csv"),
        hydro_capacities=ancient("data/bundle/hydro_capacities.csv"),
@ -478,7 +478,7 @@ rule simplify_network:
        costs=config_provider("costs"),
    input:
        network=resources("networks/elec.nc"),
-        tech_costs=COSTS,
+        tech_costs=resources(f"costs_{config['costs']['year']}.csv"),
        regions_onshore=resources("regions_onshore.geojson"),
        regions_offshore=resources("regions_offshore.geojson"),
    output:
@ -525,7 +525,7 @@ rule cluster_network:
            if config_provider("enable", "custom_busmap", default=False)(w)
            else []
        ),
-        tech_costs=COSTS,
+        tech_costs=resources(f"costs_{config['costs']['year']}.csv"),
    output:
        network=resources("networks/elec_s{simpl}_{clusters}.nc"),
        regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
@ -552,7 +552,7 @@ rule add_extra_components:
        costs=config_provider("costs"),
    input:
        network=resources("networks/elec_s{simpl}_{clusters}.nc"),
-        tech_costs=COSTS,
+        tech_costs=resources(f"costs_{config['costs']['year']}.csv"),
    output:
        resources("networks/elec_s{simpl}_{clusters}_ec.nc"),
    log:
@ -570,11 +570,11 @@ rule add_extra_components:

 rule prepare_network:
    params:
-        snapshots={
-            "resolution": config_provider("snapshots", "resolution", default=False),
+        snapshots=lambda w: {
+            "resolution": config_provider("snapshots", "resolution", default=False)(w),
            "segmentation": config_provider(
                "snapshots", "segmentation", default=False
-            ),
+            )(w),
        },
        links=config_provider("links"),
        lines=config_provider("lines"),
@ -588,7 +588,7 @@ rule prepare_network:
        autarky=config_provider("electricity", "autarky", default={}),
    input:
        resources("networks/elec_s{simpl}_{clusters}_ec.nc"),
-        tech_costs=COSTS,
+        tech_costs=resources(f"costs_{config['costs']['year']}.csv"),
        co2_price=lambda w: resources("co2_price.csv") if "Ept" in w.opts else [],
    output:
        resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"),
--- a/rules/build_sector.smk
+++ b/rules/build_sector.smk
@ -857,7 +857,7 @@ rule prepare_sector_network:
        biomass_potentials=lambda w: (
            resources(
                "biomass_potentials_s{simpl}_{clusters}_"
-                + "{}.csv".format(config_provider("biomass", "year"))
+                + "{}.csv".format(config_provider("biomass", "year")(w))
            )
            if config_provider("foresight")(w) == "overnight"
            else resources(
@ -865,9 +865,9 @@ rule prepare_sector_network:
            )
        ),
        costs=lambda w: (
-            "resources/costs_{}.csv".format(config_provider("costs", "year"))
+            resources("costs_{}.csv".format(config_provider("costs", "year")(w)))
            if config_provider("foresight")(w) == "overnight"
-            else "resources/costs_{planning_horizons}.csv"
+            else resources("costs_{planning_horizons}.csv")
        ),
        profile_offwind_ac=resources("profile_offwind-ac.nc"),
        profile_offwind_dc=resources("profile_offwind-dc.nc"),
--- a/rules/postprocess.smk
+++ b/rules/postprocess.smk
@ -18,7 +18,7 @@ if config["foresight"] != "perfect":
                "regions_onshore_elec_s{simpl}_{clusters}.geojson"
            ),
        output:
-            map=RESULTS + "maps/power-network-s{simpl}-{clusters}.pdf",
+            map=resources("maps/power-network-s{simpl}-{clusters}.pdf"),
        threads: 1
        resources:
            mem_mb=4000,
@ -149,8 +149,6 @@ rule copy_config:
    threads: 1
    resources:
        mem_mb=1000,
-    benchmark:
-        benchmarks("copy_config")
    conda:
        "../envs/environment.yaml"
    script:
@ -168,10 +166,6 @@ rule make_summary:
        scenario=config_provider("scenario"),
        RDIR=RDIR,
    input:
-        expand(
-            RESULTS + "maps/power-network-s{simpl}-{clusters}.pdf",
-            **config["scenario"],
-        ),
        networks=expand(
            RESULTS
            + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
@ -179,15 +173,16 @@ rule make_summary:
            run=config["run"]["name"],
        ),
        costs=lambda w: (
-            "resources/costs_{}.csv".format(config_provider("costs", "year"))
+            resources("costs_{}.csv".format(config_provider("costs", "year")(w)))
            if config_provider("foresight")(w) == "overnight"
-            else "resources/costs_{}.csv".format(
+            else resources("costs_{}.csv".format(
                config_provider("scenario", "planning_horizons", 0)
-            )
+            ))
        ),
        ac_plot=expand(
-            RESULTS + "maps/power-network-s{simpl}-{clusters}.pdf",
+            resources("maps/power-network-s{simpl}-{clusters}.pdf"),
            **config["scenario"],
+            run=config["run"]["name"],
        ),
        costs_plot=expand(
            RESULTS
@ -235,9 +230,7 @@ rule make_summary:
    resources:
        mem_mb=10000,
    log:
-        logs("make_summary.log"),
-    benchmark:
-        benchmarks("make_summary")
+        RESULTS + "logs/make_summary.log",
    conda:
        "../envs/environment.yaml"
    script:
@ -267,9 +260,7 @@ rule plot_summary:
    resources:
        mem_mb=10000,
    log:
-        logs("plot_summary.log"),
-    benchmark:
-        benchmarks("plot_summary")
+        RESULTS + "logs/plot_summary.log",
    conda:
        "../envs/environment.yaml"
    script:
--- a/rules/retrieve.smk
+++ b/rules/retrieve.smk
@ -88,10 +88,12 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", T
        output:
            resources("costs_{year}.csv"),
        log:
-            "logs/retrieve_cost_data_{year}.log",
+            logs("retrieve_cost_data_{year}.log"),
        resources:
            mem_mb=1000,
        retries: 2
+        conda:
+            "../envs/retrieve.yaml"
        script:
            "../scripts/retrieve_cost_data.py"

@ -191,6 +193,8 @@ if config["enable"]["retrieve"]:
        resources:
            mem_mb=5000,
        retries: 2
+        conda:
+            "../envs/retrieve.yaml"
        script:
            "../scripts/retrieve_electricity_demand.py"

--- a/rules/solve_electricity.smk
+++ b/rules/solve_electricity.smk
@ -2,7 +2,6 @@
 #
 # SPDX-License-Identifier: MIT

-
 rule solve_network:
    params:
        solving=config_provider("solving"),
--- a/rules/solve_myopic.smk
+++ b/rules/solve_myopic.smk
@ -16,9 +16,9 @@ rule add_existing_baseyear:
        busmap_s=resources("busmap_elec_s{simpl}.csv"),
        busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"),
        clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"),
-        costs=lambda w: "resources/costs_{}.csv".format(
+        costs=lambda w: resources("costs_{}.csv".format(
            config_provider("scenario", "planning_horizons", 0)(w)
-        ),
+        )),
        cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"),
        cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"),
        existing_heating_distribution=resources(
@ -76,7 +76,7 @@ rule add_brownfield:
        network=RESULTS
        + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
        network_p=solved_previous_horizon,  #solved network at previous time step
-        costs="resources/costs_{planning_horizons}.csv",
+        costs=resources("costs_{planning_horizons}.csv"),
        cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"),
        cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"),
    output:
@ -114,7 +114,7 @@ rule solve_sector_network_myopic:
    input:
        network=RESULTS
        + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-        costs="resources/costs_{planning_horizons}.csv",
+        costs=resources("costs_{planning_horizons}.csv"),
        config=RESULTS + "config.yaml",
    output:
        RESULTS
--- a/rules/solve_perfect.smk
+++ b/rules/solve_perfect.smk
@ -14,9 +14,9 @@ rule add_existing_baseyear:
        busmap_s=resources("busmap_elec_s{simpl}.csv"),
        busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"),
        clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"),
-        costs="resources/costs_{}.csv".format(
+        costs=resources("costs_{}.csv".format(
            config_provider("scenario", "planning_horizons", 0)
-        ),
+        )),
        cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"),
        cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"),
        existing_heating_distribution=resources(
@ -98,7 +98,7 @@ rule solve_sector_network_perfect:
    input:
        network=RESULTS
        + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
-        costs="resources/costs_2030.csv",
+        costs=resources("costs_2030.csv"),
        config=RESULTS + "config.yaml",
    output:
        RESULTS
@ -140,7 +140,7 @@ def input_networks_make_summary_perfect(w):
 rule make_summary_perfect:
    input:
        unpack(input_networks_make_summary_perfect),
-        costs="resources/costs_2020.csv",
+        costs=resources("costs_2020.csv"),
    output:
        nodal_costs=RESULTS + "csvs/nodal_costs.csv",
        nodal_capacities=RESULTS + "csvs/nodal_capacities.csv",
--- a/scripts/_helpers.py
+++ b/scripts/_helpers.py
@ -52,15 +52,16 @@ def get_run_path(fn, dir, rdir, shared_resources):

    Notes
    -----
-    Special case for "base" allows no wildcards other than
-    "technology" and excludes filenames starting with "networks/elec" or
+    Special case for "base" allows no wildcards other than "technology", "year"
+    and "scope" and excludes filenames starting with "networks/elec" or
    "add_electricity".
    """
    pattern = r"\{([^{}]+)\}"
-    existing_wildcards = list(re.findall(pattern, fn))
+    existing_wildcards = set(re.findall(pattern, fn))
    if shared_resources == "base":
        # special case for shared "base" resources
-        no_relevant_wildcards = not len(set(existing_wildcards) - {"technology"})
+        irrelevant_wildcards = {"technology", "year", "scope"}
+        no_relevant_wildcards = not len(existing_wildcards - irrelevant_wildcards)
        no_elec_rule = not fn.startswith("networks/elec") and not fn.startswith(
            "add_electricity"
        )
@ -68,7 +69,7 @@ def get_run_path(fn, dir, rdir, shared_resources):
    elif isinstance(shared_resources, (str, list)):
        if isinstance(shared_resources, str):
            shared_resources = [shared_resources]
-        is_shared = set(existing_wildcards).issubset(shared_resources)
+        is_shared = (existing_wildcards).issubset(shared_resources)
    else:
        is_shared = shared_resources