bjoern/pypsa-eur
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

allow_missing in collection rules, fixes for myopic, wildcard constraints without function

Browse Source
This commit is contained in:
Fabian Neumann 2024-02-17 23:36:46 +01:00
parent b543bf37c9
commit 945180f153
11 changed files with 79 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config/config.default.yaml
View File

@ -25,7 +25,7 @@ run:
enable: false
file: config/scenarios.yaml
disable_progressbar: false
shared_resources: false # TODO: splitting resources by wildcard does not work well, neither does true, only base works well
shared_resources: false
shared_cutouts: true
# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight

12
rules/build_sector.smk
View File

@ -791,8 +791,6 @@ rule build_existing_heating_distribution:
existing_heating_distribution=resources(
"existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv"
),
wildcard_constraints:
planning_horizons=config_provider("scenario", "planning_horizons", 0), #only applies to baseyear
threads: 1
resources:
mem_mb=2000,
@ -934,12 +932,12 @@ rule prepare_sector_network:
resources:
mem_mb=2000,
log:
logs(
"prepare_sector_network_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"
),
RESULTS
+ "logs/prepare_sector_network_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
benchmark:
benchmarks(
"prepare_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
(
RESULTS
+ "benchmarks/prepare_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
conda:
"../envs/environment.yaml"

7
rules/common.smk
View File

@ -136,10 +136,11 @@ def input_eurostat(w):
return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition"
def solved_previous_horizon(wildcards):
planning_horizons = config_provider("scenario", "planning_horizons")
i = planning_horizons.index(int(wildcards.planning_horizons))
def solved_previous_horizon(w):
planning_horizons = config_provider("scenario", "planning_horizons")(w)
i = planning_horizons.index(int(w.planning_horizons))
planning_horizon_p = str(planning_horizons[i - 1])
return (
RESULTS
+ "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_"

45
rules/postprocess.smk
View File

@ -43,12 +43,12 @@ if config["foresight"] != "perfect":
resources:
mem_mb=10000,
log:
logs(
"plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"
),
RESULTS
+ "logs/plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
benchmark:
benchmarks(
"plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
(
RESULTS
+ "benchmarksplot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
conda:
"../envs/environment.yaml"
@ -70,12 +70,12 @@ if config["foresight"] != "perfect":
resources:
mem_mb=10000,
log:
logs(
"plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"
),
RESULTS
+ "logs/plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
benchmark:
benchmarks(
"plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
(
RESULTS
+ "benchmarks/plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
conda:
"../envs/environment.yaml"
@ -96,12 +96,12 @@ if config["foresight"] != "perfect":
resources:
mem_mb=10000,
log:
logs(
"plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"
),
RESULTS
+ "logs/plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
benchmark:
benchmarks(
"plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
(
RESULTS
+ "benchmarks/plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
conda:
"../envs/environment.yaml"
@ -131,10 +131,6 @@ if config["foresight"] == "perfect":
threads: 2
resources:
mem_mb=10000,
benchmark:
benchmarks(
"postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_benchmark"
)
conda:
"../envs/environment.yaml"
script:
@ -167,7 +163,7 @@ rule make_summary:
RESULTS
+ "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
**config["scenario"],
run=config["run"]["name"],
allow_missing=True,
),
costs=lambda w: (
resources("costs_{}.csv".format(config_provider("costs", "year")(w)))
@ -181,13 +177,13 @@ rule make_summary:
ac_plot=expand(
resources("maps/power-network-s{simpl}-{clusters}.pdf"),
**config["scenario"],
run=config["run"]["name"],
allow_missing=True,
),
costs_plot=expand(
RESULTS
+ "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf",
**config["scenario"],
run=config["run"]["name"],
allow_missing=True,
),
h2_plot=lambda w: expand(
(
@ -197,7 +193,7 @@ rule make_summary:
else []
),
**config["scenario"],
run=config["run"]["name"],
allow_missing=True,
),
ch4_plot=lambda w: expand(
(
@ -207,7 +203,7 @@ rule make_summary:
else []
),
**config["scenario"],
run=config["run"]["name"],
allow_missing=True,
),
output:
nodal_costs=RESULTS + "csvs/nodal_costs.csv",
@ -245,6 +241,7 @@ rule plot_summary:
plotting=config_provider("plotting"),
foresight=config_provider("foresight"),
co2_budget=config_provider("co2_budget"),
sector=config_provider("sector"),
RDIR=RDIR,
input:
costs=RESULTS + "csvs/costs.csv",

23
rules/solve_electricity.smk
View File

@ -19,11 +19,13 @@ rule solve_network:
network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
log:
solver=normpath(
logs("solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log")
RESULTS
+ "logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"
),
python=logs("solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log"),
python=RESULTS
+ "logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log",
benchmark:
benchmarks("solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}")
RESULTS + "benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}"
threads: solver_threads
resources:
mem_mb=memory,
@ -45,15 +47,16 @@ rule solve_operations_network:
network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc",
log:
solver=normpath(
logs(
"solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"
)
),
python=logs(
"solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log"
RESULTS
+ "logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"
),
python=RESULTS
+ "logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log",
benchmark:
benchmarks("solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}")
(
RESULTS
+ "benchmarks/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}"
)
threads: 4
resources:
mem_mb=(lambda w: 10000 + 372 * int(w.clusters)),

40
rules/solve_myopic.smk
View File

@ -33,17 +33,20 @@ rule add_existing_baseyear:
RESULTS
+ "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
wildcard_constraints:
planning_horizons=config_provider("scenario", "planning_horizons", 0), #only applies to baseyear
# TODO: The first planning_horizon needs to be aligned across scenarios
# snakemake does not support passing functions to wildcard_constraints
# reference: https://github.com/snakemake/snakemake/issues/2703
planning_horizons=config["scenario"]["planning_horizons"][0], #only applies to baseyear
threads: 1
resources:
mem_mb=2000,
log:
logs(
"add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"
),
RESULTS
+ "logs/add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
benchmark:
benchmarks(
"add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
(
RESULTS
+ "benchmarks/add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
conda:
"../envs/environment.yaml"
@ -85,12 +88,12 @@ rule add_brownfield:
resources:
mem_mb=10000,
log:
logs(
"add_brownfield_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"
),
RESULTS
+ "logs/add_brownfield_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
benchmark:
benchmarks(
"add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
(
RESULTS
+ "benchmarks/add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
conda:
"../envs/environment.yaml"
@ -121,19 +124,18 @@ rule solve_sector_network_myopic:
shadow:
"shallow"
log:
solver=logs(
"elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log"
),
python=logs(
"elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log"
),
solver=RESULTS
+ "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log",
python=RESULTS
+ "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log",
threads: solver_threads
resources:
mem_mb=config_provider("solving", "mem"),
walltime=config_provider("solving", "walltime", default="12:00:00"),
benchmark:
benchmarks(
"solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
(
RESULTS
+ "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
conda:
"../envs/environment.yaml"

5
rules/solve_overnight.smk
View File

@ -33,8 +33,9 @@ rule solve_sector_network:
mem_mb=config_provider("solving", "mem"),
walltime=config_provider("solving", "walltime", default="12:00:00"),
benchmark:
benchmarks(
"solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
(
RESULTS
+ "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
conda:
"../envs/environment.yaml"

13
rules/solve_perfect.smk
View File

@ -32,7 +32,7 @@ rule add_existing_baseyear:
RESULTS
+ "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
wildcard_constraints:
planning_horizons=config_provider("scenario", "planning_horizons", 0), #only applies to baseyear
planning_horizons=config["scenario"]["planning_horizons"][0], #only applies to baseyear
threads: 1
resources:
mem_mb=2000,
@ -53,7 +53,8 @@ rule add_existing_baseyear:
def input_network_year(w):
return {
f"network_{year}": RESULTS
+ "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{year}.nc"
+ "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}"
+ f"_{year}.nc"
for year in config_provider("scenario", "planning_horizons")(w)[1:]
}
@ -61,6 +62,7 @@ def input_network_year(w):
rule prepare_perfect_foresight:
params:
costs=config_provider("costs"),
time_resolution=config_provider("clustering", "temporal", "sector"),
input:
unpack(input_network_year),
brownfield_network=lambda w: (
@ -98,7 +100,7 @@ rule solve_sector_network_perfect:
sector=config_provider("sector"),
planning_horizons=config_provider("scenario", "planning_horizons"),
co2_sequestration_potential=config_provider(
"sector", "co2_sequestration_potential", 200
"sector", "co2_sequestration_potential", default=200
),
custom_extra_functionality=input_custom_extra_functionality,
input:
@ -122,8 +124,9 @@ rule solve_sector_network_perfect:
memory=RESULTS
+ "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_memory.log",
benchmark:
benchmarks(
"solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years}"
(
RESULTS
+ "benchmarks/solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years}"
)
conda:
"../envs/environment.yaml"

2
scripts/base_network.py
View File

@ -767,7 +767,7 @@ if __name__ == "__main__":
if "snakemake" not in globals():
from _helpers import mock_snakemake
snakemake = mock_snakemake("base_network", run="network2019")
snakemake = mock_snakemake("base_network")
configure_logging(snakemake)
set_scenario_config(snakemake)

2
scripts/build_electricity_demand.py
View File

@ -258,7 +258,7 @@ if __name__ == "__main__":
if "snakemake" not in globals():
from _helpers import mock_snakemake
snakemake = mock_snakemake("build_electricity_demand", run="network2019")
snakemake = mock_snakemake("build_electricity_demand")
configure_logging(snakemake)
set_scenario_config(snakemake)

2
scripts/prepare_perfect_foresight.py
View File

@ -517,7 +517,7 @@ if __name__ == "__main__":
# temporal aggregate
solver_name = snakemake.config["solving"]["solver"]["name"]
segments = snakemake.params["clustering"]["temporal"]["resolution_sector"]
segments = snakemake.params.time_resolution
if isinstance(segments, (int, float)):
n = apply_time_segmentation_perfect(n, segments, solver_name=solver_name)