diff --git a/.gitignore b/.gitignore index f5f88861..5dafb3ce 100644 --- a/.gitignore +++ b/.gitignore @@ -27,7 +27,7 @@ doc/_build /scripts/create_scenarios.py /config/create_scenarios.py -config.yaml +config/config.yaml config/scenarios.yaml diff --git a/Snakefile b/Snakefile index 2b4fd64e..2c342efc 100644 --- a/Snakefile +++ b/Snakefile @@ -20,9 +20,6 @@ configfile: "config/config.default.yaml" configfile: "config/config.yaml" -COSTS = f"resources/costs_{config['costs']['year']}.csv" -ATLITE_NPROCESSES = config["atlite"].get("nprocesses", 4) - run = config["run"] scenarios = run.get("scenarios", {}) if run["name"]: @@ -39,16 +36,17 @@ benchmarks = path_provider("benchmarks/", RDIR, run["shared_resources"]) resources = path_provider("resources/", RDIR, run["shared_resources"]) CDIR = "" if run["shared_cutouts"] else RDIR -LOGS = "logs/" + RDIR -BENCHMARKS = "benchmarks/" + RDIR -if not (shared_resources := run.get("shared_resources")): - RESOURCES = "resources/" + RDIR -elif isinstance(shared_resources, str): - RESOURCES = "resources/" + shared_resources + "/" -else: - RESOURCES = "resources/" RESULTS = "results/" + RDIR +# TODO: this needs to be aligned with new scenario management +# if not (shared_resources := run.get("shared_resources")): +# RESOURCES = "resources/" + RDIR +# elif isinstance(shared_resources, str): +# RESOURCES = "resources/" + shared_resources + "/" +# else: +# RESOURCES = "resources/" + + localrules: purge, diff --git a/config/config.default.yaml b/config/config.default.yaml index ccd3baf8..9e169a5d 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -25,7 +25,7 @@ run: enable: false file: config/scenarios.yaml disable_progressbar: false - shared_resources: false + shared_resources: false # TODO: splitting resources by wildcard does not work well, neither does true, only base works well shared_cutouts: true # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 8876c68e..e18ae5dc 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -162,12 +162,12 @@ if config["enable"].get("build_cutout", False): output: protected("cutouts/" + CDIR + "{cutout}.nc"), log: - "logs/" + CDIR + "build_cutout/{cutout}.log", + logs(CDIR + "build_cutout/{cutout}.log"), benchmark: "benchmarks/" + CDIR + "build_cutout_{cutout}" - threads: ATLITE_NPROCESSES + threads: config["atlite"].get("nprocesses", 4) resources: - mem_mb=ATLITE_NPROCESSES * 1000, + mem_mb=config["atlite"].get("nprocesses", 4) * 1000, conda: "../envs/environment.yaml" script: @@ -249,9 +249,9 @@ rule determine_availability_matrix_MD_UA: availability_map=resources("availability_matrix_MD-UA_{technology}.png"), log: logs("determine_availability_matrix_MD_UA_{technology}.log"), - threads: ATLITE_NPROCESSES + threads: config["atlite"].get("nprocesses", 4) resources: - mem_mb=ATLITE_NPROCESSES * 5000, + mem_mb=config["atlite"].get("nprocesses", 4) * 5000, conda: "../envs/environment.yaml" script: @@ -319,9 +319,9 @@ rule build_renewable_profiles: logs("build_renewable_profile_{technology}.log"), benchmark: benchmarks("build_renewable_profiles_{technology}") - threads: ATLITE_NPROCESSES + threads: config["atlite"].get("nprocesses", 4) resources: - mem_mb=ATLITE_NPROCESSES * 5000, + mem_mb=config["atlite"].get("nprocesses", 4) * 5000, wildcard_constraints: technology="(?!hydro).*", # Any technology other than hydro conda: @@ -391,9 +391,9 @@ if config["lines"]["dynamic_line_rating"]["activate"]: logs("build_line_rating.log"), benchmark: benchmarks("build_line_rating") - threads: ATLITE_NPROCESSES + threads: config["atlite"].get("nprocesses", 4) resources: - mem_mb=ATLITE_NPROCESSES * 1000, + mem_mb=config["atlite"].get("nprocesses", 4) * 1000, conda: "../envs/environment.yaml" script: @@ -435,7 +435,7 @@ rule add_electricity: if config_provider("lines", "dynamic_line_rating", "activate")(w) else resources("networks/base.nc") ), - tech_costs=COSTS, + tech_costs=resources(f"costs_{config['costs']['year']}.csv"), regions=resources("regions_onshore.geojson"), powerplants=resources("powerplants.csv"), hydro_capacities=ancient("data/bundle/hydro_capacities.csv"), @@ -478,7 +478,7 @@ rule simplify_network: costs=config_provider("costs"), input: network=resources("networks/elec.nc"), - tech_costs=COSTS, + tech_costs=resources(f"costs_{config['costs']['year']}.csv"), regions_onshore=resources("regions_onshore.geojson"), regions_offshore=resources("regions_offshore.geojson"), output: @@ -525,7 +525,7 @@ rule cluster_network: if config_provider("enable", "custom_busmap", default=False)(w) else [] ), - tech_costs=COSTS, + tech_costs=resources(f"costs_{config['costs']['year']}.csv"), output: network=resources("networks/elec_s{simpl}_{clusters}.nc"), regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), @@ -552,7 +552,7 @@ rule add_extra_components: costs=config_provider("costs"), input: network=resources("networks/elec_s{simpl}_{clusters}.nc"), - tech_costs=COSTS, + tech_costs=resources(f"costs_{config['costs']['year']}.csv"), output: resources("networks/elec_s{simpl}_{clusters}_ec.nc"), log: @@ -570,11 +570,11 @@ rule add_extra_components: rule prepare_network: params: - snapshots={ - "resolution": config_provider("snapshots", "resolution", default=False), + snapshots=lambda w: { + "resolution": config_provider("snapshots", "resolution", default=False)(w), "segmentation": config_provider( "snapshots", "segmentation", default=False - ), + )(w), }, links=config_provider("links"), lines=config_provider("lines"), @@ -588,7 +588,7 @@ rule prepare_network: autarky=config_provider("electricity", "autarky", default={}), input: resources("networks/elec_s{simpl}_{clusters}_ec.nc"), - tech_costs=COSTS, + tech_costs=resources(f"costs_{config['costs']['year']}.csv"), co2_price=lambda w: resources("co2_price.csv") if "Ept" in w.opts else [], output: resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"), diff --git a/rules/build_sector.smk b/rules/build_sector.smk index cfd7191e..ba56564e 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -857,7 +857,7 @@ rule prepare_sector_network: biomass_potentials=lambda w: ( resources( "biomass_potentials_s{simpl}_{clusters}_" - + "{}.csv".format(config_provider("biomass", "year")) + + "{}.csv".format(config_provider("biomass", "year")(w)) ) if config_provider("foresight")(w) == "overnight" else resources( @@ -865,9 +865,9 @@ rule prepare_sector_network: ) ), costs=lambda w: ( - "resources/costs_{}.csv".format(config_provider("costs", "year")) + resources("costs_{}.csv".format(config_provider("costs", "year")(w))) if config_provider("foresight")(w) == "overnight" - else "resources/costs_{planning_horizons}.csv" + else resources("costs_{planning_horizons}.csv") ), profile_offwind_ac=resources("profile_offwind-ac.nc"), profile_offwind_dc=resources("profile_offwind-dc.nc"), diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 79634bf6..6302bb46 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -18,7 +18,7 @@ if config["foresight"] != "perfect": "regions_onshore_elec_s{simpl}_{clusters}.geojson" ), output: - map=RESULTS + "maps/power-network-s{simpl}-{clusters}.pdf", + map=resources("maps/power-network-s{simpl}-{clusters}.pdf"), threads: 1 resources: mem_mb=4000, @@ -149,8 +149,6 @@ rule copy_config: threads: 1 resources: mem_mb=1000, - benchmark: - benchmarks("copy_config") conda: "../envs/environment.yaml" script: @@ -168,10 +166,6 @@ rule make_summary: scenario=config_provider("scenario"), RDIR=RDIR, input: - expand( - RESULTS + "maps/power-network-s{simpl}-{clusters}.pdf", - **config["scenario"], - ), networks=expand( RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -179,15 +173,16 @@ rule make_summary: run=config["run"]["name"], ), costs=lambda w: ( - "resources/costs_{}.csv".format(config_provider("costs", "year")) + resources("costs_{}.csv".format(config_provider("costs", "year")(w))) if config_provider("foresight")(w) == "overnight" - else "resources/costs_{}.csv".format( + else resources("costs_{}.csv".format( config_provider("scenario", "planning_horizons", 0) - ) + )) ), ac_plot=expand( - RESULTS + "maps/power-network-s{simpl}-{clusters}.pdf", + resources("maps/power-network-s{simpl}-{clusters}.pdf"), **config["scenario"], + run=config["run"]["name"], ), costs_plot=expand( RESULTS @@ -235,9 +230,7 @@ rule make_summary: resources: mem_mb=10000, log: - logs("make_summary.log"), - benchmark: - benchmarks("make_summary") + RESULTS + "logs/make_summary.log", conda: "../envs/environment.yaml" script: @@ -267,9 +260,7 @@ rule plot_summary: resources: mem_mb=10000, log: - logs("plot_summary.log"), - benchmark: - benchmarks("plot_summary") + RESULTS + "logs/plot_summary.log", conda: "../envs/environment.yaml" script: diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 1b2513fb..9e9ff1f0 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -88,10 +88,12 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", T output: resources("costs_{year}.csv"), log: - "logs/retrieve_cost_data_{year}.log", + logs("retrieve_cost_data_{year}.log"), resources: mem_mb=1000, retries: 2 + conda: + "../envs/retrieve.yaml" script: "../scripts/retrieve_cost_data.py" @@ -191,6 +193,8 @@ if config["enable"]["retrieve"]: resources: mem_mb=5000, retries: 2 + conda: + "../envs/retrieve.yaml" script: "../scripts/retrieve_electricity_demand.py" diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index d3aa8d4c..b6a7902e 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -2,7 +2,6 @@ # # SPDX-License-Identifier: MIT - rule solve_network: params: solving=config_provider("solving"), diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index a52f17c0..8caf5201 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -16,9 +16,9 @@ rule add_existing_baseyear: busmap_s=resources("busmap_elec_s{simpl}.csv"), busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"), clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), - costs=lambda w: "resources/costs_{}.csv".format( + costs=lambda w: resources("costs_{}.csv".format( config_provider("scenario", "planning_horizons", 0)(w) - ), + )), cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"), cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"), existing_heating_distribution=resources( @@ -76,7 +76,7 @@ rule add_brownfield: network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", network_p=solved_previous_horizon, #solved network at previous time step - costs="resources/costs_{planning_horizons}.csv", + costs=resources("costs_{planning_horizons}.csv"), cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"), cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"), output: @@ -114,7 +114,7 @@ rule solve_sector_network_myopic: input: network=RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - costs="resources/costs_{planning_horizons}.csv", + costs=resources("costs_{planning_horizons}.csv"), config=RESULTS + "config.yaml", output: RESULTS diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 0d94ad9c..af5e884c 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -14,9 +14,9 @@ rule add_existing_baseyear: busmap_s=resources("busmap_elec_s{simpl}.csv"), busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"), clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), - costs="resources/costs_{}.csv".format( + costs=resources("costs_{}.csv".format( config_provider("scenario", "planning_horizons", 0) - ), + )), cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"), cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"), existing_heating_distribution=resources( @@ -98,7 +98,7 @@ rule solve_sector_network_perfect: input: network=RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", - costs="resources/costs_2030.csv", + costs=resources("costs_2030.csv"), config=RESULTS + "config.yaml", output: RESULTS @@ -140,7 +140,7 @@ def input_networks_make_summary_perfect(w): rule make_summary_perfect: input: unpack(input_networks_make_summary_perfect), - costs="resources/costs_2020.csv", + costs=resources("costs_2020.csv"), output: nodal_costs=RESULTS + "csvs/nodal_costs.csv", nodal_capacities=RESULTS + "csvs/nodal_capacities.csv", diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 1aa90168..c33d39f6 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -52,15 +52,16 @@ def get_run_path(fn, dir, rdir, shared_resources): Notes ----- - Special case for "base" allows no wildcards other than - "technology" and excludes filenames starting with "networks/elec" or + Special case for "base" allows no wildcards other than "technology", "year" + and "scope" and excludes filenames starting with "networks/elec" or "add_electricity". """ pattern = r"\{([^{}]+)\}" - existing_wildcards = list(re.findall(pattern, fn)) + existing_wildcards = set(re.findall(pattern, fn)) if shared_resources == "base": # special case for shared "base" resources - no_relevant_wildcards = not len(set(existing_wildcards) - {"technology"}) + irrelevant_wildcards = {"technology", "year", "scope"} + no_relevant_wildcards = not len(existing_wildcards - irrelevant_wildcards) no_elec_rule = not fn.startswith("networks/elec") and not fn.startswith( "add_electricity" ) @@ -68,7 +69,7 @@ def get_run_path(fn, dir, rdir, shared_resources): elif isinstance(shared_resources, (str, list)): if isinstance(shared_resources, str): shared_resources = [shared_resources] - is_shared = set(existing_wildcards).issubset(shared_resources) + is_shared = (existing_wildcards).issubset(shared_resources) else: is_shared = shared_resources