From 945180f153055036bcc67f633f39042299c76aa8 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 17 Feb 2024 23:36:46 +0100 Subject: [PATCH] allow_missing in collection rules, fixes for myopic, wildcard constraints without function --- config/config.default.yaml | 2 +- rules/build_sector.smk | 12 ++++---- rules/common.smk | 7 +++-- rules/postprocess.smk | 45 +++++++++++++--------------- rules/solve_electricity.smk | 23 +++++++------- rules/solve_myopic.smk | 40 +++++++++++++------------ rules/solve_overnight.smk | 5 ++-- rules/solve_perfect.smk | 13 ++++---- scripts/base_network.py | 2 +- scripts/build_electricity_demand.py | 2 +- scripts/prepare_perfect_foresight.py | 2 +- 11 files changed, 79 insertions(+), 74 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 403b7269..1e7c87ce 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -25,7 +25,7 @@ run: enable: false file: config/scenarios.yaml disable_progressbar: false - shared_resources: false # TODO: splitting resources by wildcard does not work well, neither does true, only base works well + shared_resources: false shared_cutouts: true # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 0c755834..9e8bcb90 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -791,8 +791,6 @@ rule build_existing_heating_distribution: existing_heating_distribution=resources( "existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), - wildcard_constraints: - planning_horizons=config_provider("scenario", "planning_horizons", 0), #only applies to baseyear threads: 1 resources: mem_mb=2000, @@ -934,12 +932,12 @@ rule prepare_sector_network: resources: mem_mb=2000, log: - logs( - "prepare_sector_network_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" - ), + RESULTS + + "logs/prepare_sector_network_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", benchmark: - benchmarks( - "prepare_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + ( + RESULTS + + "benchmarks/prepare_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" diff --git a/rules/common.smk b/rules/common.smk index ca339cec..9c41b57b 100644 --- a/rules/common.smk +++ b/rules/common.smk @@ -136,10 +136,11 @@ def input_eurostat(w): return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition" -def solved_previous_horizon(wildcards): - planning_horizons = config_provider("scenario", "planning_horizons") - i = planning_horizons.index(int(wildcards.planning_horizons)) +def solved_previous_horizon(w): + planning_horizons = config_provider("scenario", "planning_horizons")(w) + i = planning_horizons.index(int(w.planning_horizons)) planning_horizon_p = str(planning_horizons[i - 1]) + return ( RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_" diff --git a/rules/postprocess.smk b/rules/postprocess.smk index fe5a8c62..dc08699f 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -43,12 +43,12 @@ if config["foresight"] != "perfect": resources: mem_mb=10000, log: - logs( - "plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" - ), + RESULTS + + "logs/plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", benchmark: - benchmarks( - "plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + ( + RESULTS + + "benchmarksplot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" @@ -70,12 +70,12 @@ if config["foresight"] != "perfect": resources: mem_mb=10000, log: - logs( - "plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" - ), + RESULTS + + "logs/plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", benchmark: - benchmarks( - "plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + ( + RESULTS + + "benchmarks/plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" @@ -96,12 +96,12 @@ if config["foresight"] != "perfect": resources: mem_mb=10000, log: - logs( - "plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" - ), + RESULTS + + "logs/plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", benchmark: - benchmarks( - "plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + ( + RESULTS + + "benchmarks/plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" @@ -131,10 +131,6 @@ if config["foresight"] == "perfect": threads: 2 resources: mem_mb=10000, - benchmark: - benchmarks( - "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_benchmark" - ) conda: "../envs/environment.yaml" script: @@ -167,7 +163,7 @@ rule make_summary: RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", **config["scenario"], - run=config["run"]["name"], + allow_missing=True, ), costs=lambda w: ( resources("costs_{}.csv".format(config_provider("costs", "year")(w))) @@ -181,13 +177,13 @@ rule make_summary: ac_plot=expand( resources("maps/power-network-s{simpl}-{clusters}.pdf"), **config["scenario"], - run=config["run"]["name"], + allow_missing=True, ), costs_plot=expand( RESULTS + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", **config["scenario"], - run=config["run"]["name"], + allow_missing=True, ), h2_plot=lambda w: expand( ( @@ -197,7 +193,7 @@ rule make_summary: else [] ), **config["scenario"], - run=config["run"]["name"], + allow_missing=True, ), ch4_plot=lambda w: expand( ( @@ -207,7 +203,7 @@ rule make_summary: else [] ), **config["scenario"], - run=config["run"]["name"], + allow_missing=True, ), output: nodal_costs=RESULTS + "csvs/nodal_costs.csv", @@ -245,6 +241,7 @@ rule plot_summary: plotting=config_provider("plotting"), foresight=config_provider("foresight"), co2_budget=config_provider("co2_budget"), + sector=config_provider("sector"), RDIR=RDIR, input: costs=RESULTS + "csvs/costs.csv", diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index d3aa8d4c..6ddbc415 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -19,11 +19,13 @@ rule solve_network: network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", log: solver=normpath( - logs("solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log") + RESULTS + + "logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log" ), - python=logs("solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log"), + python=RESULTS + + "logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", benchmark: - benchmarks("solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}") + RESULTS + "benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: solver_threads resources: mem_mb=memory, @@ -45,15 +47,16 @@ rule solve_operations_network: network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc", log: solver=normpath( - logs( - "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log" - ) - ), - python=logs( - "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log" + RESULTS + + "logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log" ), + python=RESULTS + + "logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", benchmark: - benchmarks("solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}") + ( + RESULTS + + "benchmarks/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" + ) threads: 4 resources: mem_mb=(lambda w: 10000 + 372 * int(w.clusters)), diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 8574bbb8..5eb30233 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -33,17 +33,20 @@ rule add_existing_baseyear: RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", wildcard_constraints: - planning_horizons=config_provider("scenario", "planning_horizons", 0), #only applies to baseyear + # TODO: The first planning_horizon needs to be aligned across scenarios + # snakemake does not support passing functions to wildcard_constraints + # reference: https://github.com/snakemake/snakemake/issues/2703 + planning_horizons=config["scenario"]["planning_horizons"][0], #only applies to baseyear threads: 1 resources: mem_mb=2000, log: - logs( - "add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" - ), + RESULTS + + "logs/add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", benchmark: - benchmarks( - "add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + ( + RESULTS + + "benchmarks/add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" @@ -85,12 +88,12 @@ rule add_brownfield: resources: mem_mb=10000, log: - logs( - "add_brownfield_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" - ), + RESULTS + + "logs/add_brownfield_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", benchmark: - benchmarks( - "add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + ( + RESULTS + + "benchmarks/add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" @@ -121,19 +124,18 @@ rule solve_sector_network_myopic: shadow: "shallow" log: - solver=logs( - "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log" - ), - python=logs( - "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log" - ), + solver=RESULTS + + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log", + python=RESULTS + + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log", threads: solver_threads resources: mem_mb=config_provider("solving", "mem"), walltime=config_provider("solving", "walltime", default="12:00:00"), benchmark: - benchmarks( - "solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + ( + RESULTS + + "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index 64ad007c..69e2f364 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -33,8 +33,9 @@ rule solve_sector_network: mem_mb=config_provider("solving", "mem"), walltime=config_provider("solving", "walltime", default="12:00:00"), benchmark: - benchmarks( - "solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + ( + RESULTS + + "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: "../envs/environment.yaml" diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 639bcae8..d3badcb0 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -32,7 +32,7 @@ rule add_existing_baseyear: RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", wildcard_constraints: - planning_horizons=config_provider("scenario", "planning_horizons", 0), #only applies to baseyear + planning_horizons=config["scenario"]["planning_horizons"][0], #only applies to baseyear threads: 1 resources: mem_mb=2000, @@ -53,7 +53,8 @@ rule add_existing_baseyear: def input_network_year(w): return { f"network_{year}": RESULTS - + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{year}.nc" + + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}" + + f"_{year}.nc" for year in config_provider("scenario", "planning_horizons")(w)[1:] } @@ -61,6 +62,7 @@ def input_network_year(w): rule prepare_perfect_foresight: params: costs=config_provider("costs"), + time_resolution=config_provider("clustering", "temporal", "sector"), input: unpack(input_network_year), brownfield_network=lambda w: ( @@ -98,7 +100,7 @@ rule solve_sector_network_perfect: sector=config_provider("sector"), planning_horizons=config_provider("scenario", "planning_horizons"), co2_sequestration_potential=config_provider( - "sector", "co2_sequestration_potential", 200 + "sector", "co2_sequestration_potential", default=200 ), custom_extra_functionality=input_custom_extra_functionality, input: @@ -122,8 +124,9 @@ rule solve_sector_network_perfect: memory=RESULTS + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_memory.log", benchmark: - benchmarks( - "solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years}" + ( + RESULTS + + "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years}" ) conda: "../envs/environment.yaml" diff --git a/scripts/base_network.py b/scripts/base_network.py index d83e0588..515ae481 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -767,7 +767,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("base_network", run="network2019") + snakemake = mock_snakemake("base_network") configure_logging(snakemake) set_scenario_config(snakemake) diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py index 4415c835..47befb8a 100755 --- a/scripts/build_electricity_demand.py +++ b/scripts/build_electricity_demand.py @@ -258,7 +258,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("build_electricity_demand", run="network2019") + snakemake = mock_snakemake("build_electricity_demand") configure_logging(snakemake) set_scenario_config(snakemake) diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index 2f2ff608..7f312a8f 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -517,7 +517,7 @@ if __name__ == "__main__": # temporal aggregate solver_name = snakemake.config["solving"]["solver"]["name"] - segments = snakemake.params["clustering"]["temporal"]["resolution_sector"] + segments = snakemake.params.time_resolution if isinstance(segments, (int, float)): n = apply_time_segmentation_perfect(n, segments, solver_name=solver_name)