bjoern/pypsa-eur
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

allow_missing in collection rules, fixes for myopic, wildcard constraints without function

Browse Source
This commit is contained in:
Fabian Neumann 2024-02-17 23:36:46 +01:00
parent b543bf37c9
commit 945180f153
11 changed files with 79 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config/config.default.yaml
View File

@ -25,7 +25,7 @@ run:
enable: false enable: false
file: config/scenarios.yaml file: config/scenarios.yaml
disable_progressbar: false disable_progressbar: false
shared_resources: false # TODO: splitting resources by wildcard does not work well, neither does true, only base works well shared_resources: false
shared_cutouts: true shared_cutouts: true
# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight

12
rules/build_sector.smk
View File

@ -791,8 +791,6 @@ rule build_existing_heating_distribution:
existing_heating_distribution=resources( existing_heating_distribution=resources(
"existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv" "existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv"
), ),
wildcard_constraints:
planning_horizons=config_provider("scenario", "planning_horizons", 0), #only applies to baseyear
threads: 1 threads: 1
resources: resources:
mem_mb=2000, mem_mb=2000,
@ -934,12 +932,12 @@ rule prepare_sector_network:
resources: resources:
mem_mb=2000, mem_mb=2000,
log: log:
logs( RESULTS
"prepare_sector_network_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" + "logs/prepare_sector_network_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
),
benchmark: benchmark:
benchmarks( (
"prepare_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" RESULTS
+ "benchmarks/prepare_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
) )
conda: conda:
"../envs/environment.yaml" "../envs/environment.yaml"

7
rules/common.smk
View File

@ -136,10 +136,11 @@ def input_eurostat(w):
return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition" return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition"
def solved_previous_horizon(wildcards): def solved_previous_horizon(w):
planning_horizons = config_provider("scenario", "planning_horizons") planning_horizons = config_provider("scenario", "planning_horizons")(w)
i = planning_horizons.index(int(wildcards.planning_horizons)) i = planning_horizons.index(int(w.planning_horizons))
planning_horizon_p = str(planning_horizons[i - 1]) planning_horizon_p = str(planning_horizons[i - 1])
return ( return (
RESULTS RESULTS
+ "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_" + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_"

45
rules/postprocess.smk
View File

@ -43,12 +43,12 @@ if config["foresight"] != "perfect":
resources: resources:
mem_mb=10000, mem_mb=10000,
log: log:
logs( RESULTS
"plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" + "logs/plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
),
benchmark: benchmark:
benchmarks( (
"plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" RESULTS
+ "benchmarksplot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
) )
conda: conda:
"../envs/environment.yaml" "../envs/environment.yaml"
@ -70,12 +70,12 @@ if config["foresight"] != "perfect":
resources: resources:
mem_mb=10000, mem_mb=10000,
log: log:
logs( RESULTS
"plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" + "logs/plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
),
benchmark: benchmark:
benchmarks( (
"plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" RESULTS
+ "benchmarks/plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
) )
conda: conda:
"../envs/environment.yaml" "../envs/environment.yaml"
@ -96,12 +96,12 @@ if config["foresight"] != "perfect":
resources: resources:
mem_mb=10000, mem_mb=10000,
log: log:
logs( RESULTS
"plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" + "logs/plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
),
benchmark: benchmark:
benchmarks( (
"plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" RESULTS
+ "benchmarks/plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
) )
conda: conda:
"../envs/environment.yaml" "../envs/environment.yaml"
@ -131,10 +131,6 @@ if config["foresight"] == "perfect":
threads: 2 threads: 2
resources: resources:
mem_mb=10000, mem_mb=10000,
benchmark:
benchmarks(
"postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_benchmark"
)
conda: conda:
"../envs/environment.yaml" "../envs/environment.yaml"
script: script:
@ -167,7 +163,7 @@ rule make_summary:
RESULTS RESULTS
+ "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
**config["scenario"], **config["scenario"],
run=config["run"]["name"], allow_missing=True,
), ),
costs=lambda w: ( costs=lambda w: (
resources("costs_{}.csv".format(config_provider("costs", "year")(w))) resources("costs_{}.csv".format(config_provider("costs", "year")(w)))
@ -181,13 +177,13 @@ rule make_summary:
ac_plot=expand( ac_plot=expand(
resources("maps/power-network-s{simpl}-{clusters}.pdf"), resources("maps/power-network-s{simpl}-{clusters}.pdf"),
**config["scenario"], **config["scenario"],
run=config["run"]["name"], allow_missing=True,
), ),
costs_plot=expand( costs_plot=expand(
RESULTS RESULTS
+ "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf",
**config["scenario"], **config["scenario"],
run=config["run"]["name"], allow_missing=True,
), ),
h2_plot=lambda w: expand( h2_plot=lambda w: expand(
( (
@ -197,7 +193,7 @@ rule make_summary:
else [] else []
), ),
**config["scenario"], **config["scenario"],
run=config["run"]["name"], allow_missing=True,
), ),
ch4_plot=lambda w: expand( ch4_plot=lambda w: expand(
( (
@ -207,7 +203,7 @@ rule make_summary:
else [] else []
), ),
**config["scenario"], **config["scenario"],
run=config["run"]["name"], allow_missing=True,
), ),
output: output:
nodal_costs=RESULTS + "csvs/nodal_costs.csv", nodal_costs=RESULTS + "csvs/nodal_costs.csv",
@ -245,6 +241,7 @@ rule plot_summary:
plotting=config_provider("plotting"), plotting=config_provider("plotting"),
foresight=config_provider("foresight"), foresight=config_provider("foresight"),
co2_budget=config_provider("co2_budget"), co2_budget=config_provider("co2_budget"),
sector=config_provider("sector"),
RDIR=RDIR, RDIR=RDIR,
input: input:
costs=RESULTS + "csvs/costs.csv", costs=RESULTS + "csvs/costs.csv",

23
rules/solve_electricity.smk
View File

@ -19,11 +19,13 @@ rule solve_network:
network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
log: log:
solver=normpath( solver=normpath(
logs("solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log") RESULTS
+ "logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"
), ),
python=logs("solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log"), python=RESULTS
+ "logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log",
benchmark: benchmark:
benchmarks("solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}") RESULTS + "benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}"
threads: solver_threads threads: solver_threads
resources: resources:
mem_mb=memory, mem_mb=memory,
@ -45,15 +47,16 @@ rule solve_operations_network:
network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc", network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc",
log: log:
solver=normpath( solver=normpath(
logs( RESULTS
"solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log" + "logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"
)
),
python=logs(
"solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log"
), ),
python=RESULTS
+ "logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log",
benchmark: benchmark:
benchmarks("solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}") (
RESULTS
+ "benchmarks/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}"
)
threads: 4 threads: 4
resources: resources:
mem_mb=(lambda w: 10000 + 372 * int(w.clusters)), mem_mb=(lambda w: 10000 + 372 * int(w.clusters)),

40
rules/solve_myopic.smk
View File

@ -33,17 +33,20 @@ rule add_existing_baseyear:
RESULTS RESULTS
+ "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
wildcard_constraints: wildcard_constraints:
planning_horizons=config_provider("scenario", "planning_horizons", 0), #only applies to baseyear # TODO: The first planning_horizon needs to be aligned across scenarios
# snakemake does not support passing functions to wildcard_constraints
# reference: https://github.com/snakemake/snakemake/issues/2703
planning_horizons=config["scenario"]["planning_horizons"][0], #only applies to baseyear
threads: 1 threads: 1
resources: resources:
mem_mb=2000, mem_mb=2000,
log: log:
logs( RESULTS
"add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" + "logs/add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
),
benchmark: benchmark:
benchmarks( (
"add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" RESULTS
+ "benchmarks/add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
) )
conda: conda:
"../envs/environment.yaml" "../envs/environment.yaml"
@ -85,12 +88,12 @@ rule add_brownfield:
resources: resources:
mem_mb=10000, mem_mb=10000,
log: log:
logs( RESULTS
"add_brownfield_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" + "logs/add_brownfield_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
),
benchmark: benchmark:
benchmarks( (
"add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" RESULTS
+ "benchmarks/add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
) )
conda: conda:
"../envs/environment.yaml" "../envs/environment.yaml"
@ -121,19 +124,18 @@ rule solve_sector_network_myopic:
shadow: shadow:
"shallow" "shallow"
log: log:
solver=logs( solver=RESULTS
"elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log" + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log",
), python=RESULTS
python=logs( + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log",
"elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log"
),
threads: solver_threads threads: solver_threads
resources: resources:
mem_mb=config_provider("solving", "mem"), mem_mb=config_provider("solving", "mem"),
walltime=config_provider("solving", "walltime", default="12:00:00"), walltime=config_provider("solving", "walltime", default="12:00:00"),
benchmark: benchmark:
benchmarks( (
"solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" RESULTS
+ "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
) )
conda: conda:
"../envs/environment.yaml" "../envs/environment.yaml"

5
rules/solve_overnight.smk
View File

@ -33,8 +33,9 @@ rule solve_sector_network:
mem_mb=config_provider("solving", "mem"), mem_mb=config_provider("solving", "mem"),
walltime=config_provider("solving", "walltime", default="12:00:00"), walltime=config_provider("solving", "walltime", default="12:00:00"),
benchmark: benchmark:
benchmarks( (
"solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" RESULTS
+ "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
) )
conda: conda:
"../envs/environment.yaml" "../envs/environment.yaml"

13
rules/solve_perfect.smk
View File

@ -32,7 +32,7 @@ rule add_existing_baseyear:
RESULTS RESULTS
+ "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
wildcard_constraints: wildcard_constraints:
planning_horizons=config_provider("scenario", "planning_horizons", 0), #only applies to baseyear planning_horizons=config["scenario"]["planning_horizons"][0], #only applies to baseyear
threads: 1 threads: 1
resources: resources:
mem_mb=2000, mem_mb=2000,
@ -53,7 +53,8 @@ rule add_existing_baseyear:
def input_network_year(w): def input_network_year(w):
return { return {
f"network_{year}": RESULTS f"network_{year}": RESULTS
+ "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{year}.nc" + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}"
+ f"_{year}.nc"
for year in config_provider("scenario", "planning_horizons")(w)[1:] for year in config_provider("scenario", "planning_horizons")(w)[1:]
} }
@ -61,6 +62,7 @@ def input_network_year(w):
rule prepare_perfect_foresight: rule prepare_perfect_foresight:
params: params:
costs=config_provider("costs"), costs=config_provider("costs"),
time_resolution=config_provider("clustering", "temporal", "sector"),
input: input:
unpack(input_network_year), unpack(input_network_year),
brownfield_network=lambda w: ( brownfield_network=lambda w: (
@ -98,7 +100,7 @@ rule solve_sector_network_perfect:
sector=config_provider("sector"), sector=config_provider("sector"),
planning_horizons=config_provider("scenario", "planning_horizons"), planning_horizons=config_provider("scenario", "planning_horizons"),
co2_sequestration_potential=config_provider( co2_sequestration_potential=config_provider(
"sector", "co2_sequestration_potential", 200 "sector", "co2_sequestration_potential", default=200
), ),
custom_extra_functionality=input_custom_extra_functionality, custom_extra_functionality=input_custom_extra_functionality,
input: input:
@ -122,8 +124,9 @@ rule solve_sector_network_perfect:
memory=RESULTS memory=RESULTS
+ "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_memory.log", + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_memory.log",
benchmark: benchmark:
benchmarks( (
"solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years}" RESULTS
+ "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years}"
) )
conda: conda:
"../envs/environment.yaml" "../envs/environment.yaml"

2
scripts/base_network.py
View File

@ -767,7 +767,7 @@ if __name__ == "__main__":
if "snakemake" not in globals(): if "snakemake" not in globals():
from _helpers import mock_snakemake from _helpers import mock_snakemake
snakemake = mock_snakemake("base_network", run="network2019") snakemake = mock_snakemake("base_network")
configure_logging(snakemake) configure_logging(snakemake)
set_scenario_config(snakemake) set_scenario_config(snakemake)

2
scripts/build_electricity_demand.py
View File

@ -258,7 +258,7 @@ if __name__ == "__main__":
if "snakemake" not in globals(): if "snakemake" not in globals():
from _helpers import mock_snakemake from _helpers import mock_snakemake
snakemake = mock_snakemake("build_electricity_demand", run="network2019") snakemake = mock_snakemake("build_electricity_demand")
configure_logging(snakemake) configure_logging(snakemake)
set_scenario_config(snakemake) set_scenario_config(snakemake)

2
scripts/prepare_perfect_foresight.py
View File

@ -517,7 +517,7 @@ if __name__ == "__main__":
# temporal aggregate # temporal aggregate
solver_name = snakemake.config["solving"]["solver"]["name"] solver_name = snakemake.config["solving"]["solver"]["name"]
segments = snakemake.params["clustering"]["temporal"]["resolution_sector"] segments = snakemake.params.time_resolution
if isinstance(segments, (int, float)): if isinstance(segments, (int, float)):
n = apply_time_segmentation_perfect(n, segments, solver_name=solver_name) n = apply_time_segmentation_perfect(n, segments, solver_name=solver_name)