bjoern/pypsa-eur
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

add new resources(), logs(), benchmarks(), config_providers()

Browse Source
This commit is contained in:
Fabian Neumann 2024-02-10 18:09:23 +01:00
parent e838b63ded
commit a9dad3f34e
12 changed files with 163 additions and 238 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Snakefile
View File

@ -91,7 +91,7 @@ if config["foresight"] == "perfect":
rule all:
input:
RESULTS + "graphs/costs.pdf",
expand(RESULTS + "graphs/costs.pdf", run=config["run"]["name"]),
default_target: True

63
rules/build_electricity.smk
View File

@ -20,7 +20,7 @@ if config["enable"].get("prepare_links_p_nom", False):
rule build_electricity_demand:
params:
snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, # TODO: use config provider
snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]},
countries=config_provider("countries"),
load=config_provider("load"),
input:
@ -62,7 +62,7 @@ rule build_powerplants:
rule base_network:
params:
countries=config_provider("countries"),
snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, # TODO: use config provider
snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]},
lines=config_provider("lines"),
links=config_provider("links"),
transformers=config_provider("transformers"),
@ -145,7 +145,7 @@ if config["enable"].get("build_cutout", False):
rule build_cutout:
params:
snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, # TODO: use config provider
snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]},
cutouts=config_provider("atlite", "cutouts"),
input:
regions_onshore=resources("regions_onshore.geojson"),
@ -170,7 +170,7 @@ if config["enable"].get("build_natura_raster", False):
rule build_natura_raster:
input:
natura=ancient("data/bundle/natura/Natura2000_end2015.shp"),
cutouts=expand("cutouts/" + CDIR + "{cutouts}.nc", **config["atlite"]),
cutouts=expand("cutouts/" + CDIR + "{cutouts}.nc", **config_provider("atlite")),
output:
resources("natura.tiff"),
resources:
@ -189,8 +189,8 @@ rule build_ship_raster:
cutouts=expand(
"cutouts/" + CDIR + "{cutout}.nc",
cutout=[
config["renewable"][k]["cutout"]
for k in config["electricity"]["renewable_carriers"]
config_provider("renewable", k, "cutout")
for k in config_provider("electricity", "renewable_carriers")
],
),
output:
@ -214,30 +214,30 @@ rule determine_availability_matrix_MD_UA:
wdpa_marine="data/WDPA_WDOECM_marine.gpkg",
gebco=lambda w: (
"data/bundle/GEBCO_2014_2D.nc"
if "max_depth" in config["renewable"][w.technology].keys()
if config_provider("renewable", w.technology)(w).get("max_depth")
else []
),
ship_density=lambda w: (
RESOURCES + "shipdensity_raster.tif"
if "ship_threshold" in config["renewable"][w.technology].keys()
resources("shipdensity_raster.tif")
if "ship_threshold" in config_provider("renewable", w.technology)(w).keys()
else []
),
country_shapes=RESOURCES + "country_shapes.geojson",
offshore_shapes=RESOURCES + "offshore_shapes.geojson",
country_shapes=resources("country_shapes.geojson"),
offshore_shapes=resources("offshore_shapes.geojson"),
regions=lambda w: (
RESOURCES + "regions_onshore.geojson"
resources("regions_onshore.geojson")
if w.technology in ("onwind", "solar")
else RESOURCES + "regions_offshore.geojson"
else resources("regions_offshore.geojson")
),
cutout=lambda w: "cutouts/"
+ CDIR
+ config["renewable"][w.technology]["cutout"]
+ config_provider("renewable", w.technology, "cutout")(w)
+ ".nc",
output:
availability_matrix=RESOURCES + "availability_matrix_MD-UA_{technology}.nc",
availability_map=RESOURCES + "availability_matrix_MD-UA_{technology}.png",
availability_matrix=resources("availability_matrix_MD-UA_{technology}.nc"),
availability_map=resources("availability_matrix_MD-UA_{technology}.png"),
log:
LOGS + "determine_availability_matrix_MD_UA_{technology}.log",
logs("determine_availability_matrix_MD_UA_{technology}.log"),
threads: ATLITE_NPROCESSES
resources:
mem_mb=ATLITE_NPROCESSES * 5000,
@ -250,8 +250,7 @@ rule determine_availability_matrix_MD_UA:
# Optional input when having Ukraine (UA) or Moldova (MD) in the countries list
if {"UA", "MD"}.intersection(set(config["countries"])):
opt = {
"availability_matrix_MD_UA": RESOURCES
+ "availability_matrix_MD-UA_{technology}.nc"
"availability_matrix_MD_UA": resources("availability_matrix_MD-UA_{technology}.nc")
}
else:
opt = {}
@ -259,7 +258,7 @@ else:
rule build_renewable_profiles:
params:
snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, # TODO: use config provider
snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]},
renewable=config_provider("renewable"),
input:
**opt,
@ -272,7 +271,7 @@ rule build_renewable_profiles:
),
luisa=lambda w: (
"data/LUISA_basemap_020321_50m.tif"
if config["renewable"][w.technology].get("luisa")
if config_provider("renewable", w.technology, "luisa")(w)
else []
),
gebco=ancient(
@ -340,7 +339,7 @@ rule build_hydro_profile:
input:
country_shapes=resources("country_shapes.geojson"),
eia_hydro_generation="data/eia_hydro_annual_generation.csv",
cutout=f"cutouts/" + CDIR + config["renewable"]["hydro"]["cutout"] + ".nc",
cutout=f"cutouts/" + CDIR + config_provider("renewable", "hydro", "cutout") + ".nc",
output:
resources("profile_hydro.nc"),
log:
@ -357,12 +356,12 @@ if config["lines"]["dynamic_line_rating"]["activate"]:
rule build_line_rating:
params:
snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]},
input:
base_network=resources("networks/base.nc"),
cutout="cutouts/"
+ CDIR
+ config["lines"]["dynamic_line_rating"]["cutout"]
+ config_provider("lines", "dynamic_line_rating", "cutout")
+ ".nc",
output:
output=resources("networks/line_rating.nc"),
@ -391,19 +390,19 @@ rule add_electricity:
input:
**{
f"profile_{tech}": resources(f"profile_{tech}.nc")
for tech in config["electricity"]["renewable_carriers"]
for tech in config_provider("electricity", "renewable_carriers")
},
**{
f"conventional_{carrier}_{attr}": fn
for carrier, d in config.get("conventional", {None: {}}).items()
if carrier in config["electricity"]["conventional_carriers"]
if carrier in config_provider("electricity", "conventional_carriers")
for attr, fn in d.items()
if str(fn).startswith("data/")
},
base_network=resources("networks/base.nc"),
line_rating=(
resources("networks/line_rating.nc")
if config["lines"]["dynamic_line_rating"]["activate"]
if config_provider("lines", "dynamic_line_rating", "activate")
else resources("networks/base.nc")
),
tech_costs=COSTS,
@ -414,7 +413,7 @@ rule add_electricity:
unit_commitment="data/unit_commitment.csv",
fuel_price=(
resources("monthly_fuel_price.csv")
if config["conventional"]["dynamic_fuel_price"]
if config_provider("conventional", "dynamic_fuel_price")
else []
),
load=resources("load.csv"),
@ -493,7 +492,7 @@ rule cluster_network:
busmap=ancient(resources("busmap_elec_s{simpl}.csv")),
custom_busmap=(
"data/custom_busmap_elec_s{simpl}_{clusters}.csv"
if config["enable"].get("custom_busmap", False)
if config_provider("enable", "custom_busmap", default=False)
else []
),
tech_costs=COSTS,
@ -542,10 +541,10 @@ rule add_extra_components:
rule prepare_network:
params:
snapshots={
"resolution": config["snapshots"].get("resolution", False),
"segmentation": config["snapshots"].get("segmentation", False),
"resolution": config_provider("snapshots", "resolution", default=False),
"segmentation": config_provider("snapshots", "segmentation", default=False),
},
# TODO: use config provider
links=config_provider("links"),
lines=config_provider("lines"),
co2base=config_provider("electricity", "co2base"),

90
rules/build_sector.smk
View File

@ -7,7 +7,7 @@ rule build_population_layouts:
input:
nuts3_shapes=resources("nuts3_shapes.geojson"),
urban_percent="data/urban_percent.csv",
cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc",
cutout="cutouts/" + CDIR + config_provider("atlite", "default_cutout") + ".nc",
output:
pop_layout_total=resources("pop_layout_total.nc"),
pop_layout_urban=resources("pop_layout_urban.nc"),
@ -31,7 +31,7 @@ rule build_clustered_population_layouts:
pop_layout_urban=resources("pop_layout_urban.nc"),
pop_layout_rural=resources("pop_layout_rural.nc"),
regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc",
cutout="cutouts/" + CDIR + config_provider("atlite", "default_cutout") + ".nc",
output:
clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"),
log:
@ -52,7 +52,7 @@ rule build_simplified_population_layouts:
pop_layout_urban=resources("pop_layout_urban.nc"),
pop_layout_rural=resources("pop_layout_rural.nc"),
regions_onshore=resources("regions_onshore_elec_s{simpl}.geojson"),
cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc",
cutout="cutouts/" + CDIR + config_provider("atlite", "default_cutout") + ".nc",
output:
clustered_pop_layout=resources("pop_layout_elec_s{simpl}.csv"),
resources:
@ -126,11 +126,11 @@ rule cluster_gas_network:
rule build_daily_heat_demand:
params:
snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, # TODO: use config_provider
snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]},
input:
pop_layout=resources("pop_layout_{scope}.nc"),
regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc",
cutout="cutouts/" + CDIR + config_provider("atlite", "default_cutout") + ".nc",
output:
heat_demand=resources("daily_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc"),
resources:
@ -148,19 +148,19 @@ rule build_daily_heat_demand:
rule build_hourly_heat_demand:
params:
snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]},
input:
heat_profile="data/heat_load_profile_BDEW.csv",
heat_demand=RESOURCES + "daily_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc",
heat_demand=resources("daily_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc"),
output:
heat_demand=RESOURCES + "hourly_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc",
heat_demand=resources("hourly_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc"),
resources:
mem_mb=2000,
threads: 8
log:
LOGS + "build_hourly_heat_demand_{scope}_{simpl}_{clusters}.loc",
logs("build_hourly_heat_demand_{scope}_{simpl}_{clusters}.loc"),
benchmark:
BENCHMARKS + "build_hourly_heat_demand/{scope}_s{simpl}_{clusters}"
benchmarks("build_hourly_heat_demand/{scope}_s{simpl}_{clusters}")
conda:
"../envs/environment.yaml"
script:
@ -169,11 +169,11 @@ rule build_hourly_heat_demand:
rule build_temperature_profiles:
params:
snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, # TODO: use config_provider
snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]},
input:
pop_layout=resources("pop_layout_{scope}.nc"),
regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc",
cutout="cutouts/" + CDIR + config_provider("atlite", "default_cutout") + ".nc",
output:
temp_soil=resources("temp_soil_{scope}_elec_s{simpl}_{clusters}.nc"),
temp_air=resources("temp_air_{scope}_elec_s{simpl}_{clusters}.nc"),
@ -221,12 +221,12 @@ rule build_cop_profiles:
rule build_solar_thermal_profiles:
params:
snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, # TODO use config_provider
snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]}, # TODO use config_provider
solar_thermal=config_provider("solar_thermal"),
input:
pop_layout=resources("pop_layout_{scope}.nc"),
regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc",
cutout="cutouts/" + CDIR + config_provider("atlite", "default_cutout") + ".nc",
output:
solar_thermal=resources("solar_thermal_{scope}_elec_s{simpl}_{clusters}.nc"),
resources:
@ -711,7 +711,7 @@ rule build_shipping_demand:
rule build_transport_demand:
params:
snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, # TODO: use config_provider
snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]},
sector=config_provider("sector"),
input:
clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"),
@ -740,18 +740,17 @@ rule build_transport_demand:
rule build_district_heat_share:
params:
sector=config["sector"],
sector=config_provider("sector"),
input:
district_heat_share=RESOURCES + "district_heat_share.csv",
clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv",
district_heat_share=resources("district_heat_share.csv"),
clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"),
output:
district_heat_share=RESOURCES
+ "district_heat_share_elec_s{simpl}_{clusters}_{planning_horizons}.csv",
district_heat_share=resources("district_heat_share_elec_s{simpl}_{clusters}_{planning_horizons}.csv"),
threads: 1
resources:
mem_mb=1000,
log:
LOGS + "build_district_heat_share_s{simpl}_{clusters}_{planning_horizons}.log",
logs("build_district_heat_share_s{simpl}_{clusters}_{planning_horizons}.log"),
conda:
"../envs/environment.yaml"
script:
@ -760,32 +759,25 @@ rule build_district_heat_share:
rule build_existing_heating_distribution:
params:
baseyear=config["scenario"]["planning_horizons"][0],
sector=config["sector"],
existing_capacities=config["existing_capacities"],
baseyear=config_provider("scenario", "planning_horizons", 0),
sector=config_provider("sector"),
existing_capacities=config_provider("existing_capacities"),
input:
existing_heating="data/existing_infrastructure/existing_heating_raw.csv",
clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv",
clustered_pop_energy_layout=RESOURCES
+ "pop_weighted_energy_totals_s{simpl}_{clusters}.csv",
district_heat_share=RESOURCES
+ "district_heat_share_elec_s{simpl}_{clusters}_{planning_horizons}.csv",
clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"),
clustered_pop_energy_layout=resources("pop_weighted_energy_totals_s{simpl}_{clusters}.csv"),
district_heat_share=resources("district_heat_share_elec_s{simpl}_{clusters}_{planning_horizons}.csv"),
output:
existing_heating_distribution=RESOURCES
+ "existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv",
existing_heating_distribution=resources("existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv"),
wildcard_constraints:
planning_horizons=config["scenario"]["planning_horizons"][0], #only applies to baseyear
planning_horizons=config_provider("scenario", "planning_horizons", 0), #only applies to baseyear
threads: 1
resources:
mem_mb=2000,
log:
LOGS
+ "build_existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.log",
logs("build_existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.log"),
benchmark:
(
BENCHMARKS
+ "build_existing_heating_distribution/elec_s{simpl}_{clusters}_{planning_horizons}"
)
benchmarks("build_existing_heating_distribution/elec_s{simpl}_{clusters}_{planning_horizons}")
conda:
"../envs/environment.yaml"
script:
@ -831,16 +823,16 @@ rule prepare_sector_network:
biomass_potentials=(
resources(
"biomass_potentials_s{simpl}_{clusters}_"
+ "{}.csv".format(config["biomass"]["year"])
+ "{}.csv".format(config_provider("biomass", "year"))
)
if config["foresight"] == "overnight"
if config_provider("foresight") == "overnight"
else resources(
"biomass_potentials_s{simpl}_{clusters}_{planning_horizons}.csv"
)
),
costs=(
"data/costs_{}.csv".format(config["costs"]["year"])
if config["foresight"] == "overnight"
"data/costs_{}.csv".format(config_provider("costs", "year"))
if config_provider("foresight") == "overnight"
else "data/costs_{planning_horizons}.csv"
),
profile_offwind_ac=resources("profile_offwind-ac.nc"),
@ -873,17 +865,17 @@ rule prepare_sector_network:
cop_air_urban=resources("cop_air_urban_elec_s{simpl}_{clusters}.nc"),
solar_thermal_total=(
resources("solar_thermal_total_elec_s{simpl}_{clusters}.nc")
if config["sector"]["solar_thermal"]
if config_provider("sector", "solar_thermal")
else []
),
solar_thermal_urban=(
resources("solar_thermal_urban_elec_s{simpl}_{clusters}.nc")
if config["sector"]["solar_thermal"]
if config_provider("sector", "solar_thermal")
else []
),
solar_thermal_rural=(
resources("solar_thermal_rural_elec_s{simpl}_{clusters}.nc")
if config["sector"]["solar_thermal"]
if config_provider("sector", "solar_thermal")
else []
),
output:
@ -893,13 +885,9 @@ rule prepare_sector_network:
resources:
mem_mb=2000,
log:
LOGS
+ "prepare_sector_network_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
logs("prepare_sector_network_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"),
benchmark:
(
BENCHMARKS
+ "prepare_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
benchmarks("prepare_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}")
conda:
"../envs/environment.yaml"
script:

6
rules/collect.smk
View File

@ -13,12 +13,6 @@ localrules:
solve_sector_networks,
rule all:
input:
expand(RESULTS + "graphs/costs.pdf", run=config["run"]["name"]),
default_target: True
rule cluster_networks:
input:
expand(

10
rules/common.smk
View File

@ -78,8 +78,8 @@ def config_provider(*keys, default=None):
def solver_threads(w):
solver_options = config["solving"]["solver_options"]
option_set = config["solving"]["solver"]["options"]
solver_options = config_provider("solving", "solver_options")
option_set = config_provider("solving", "solver", "options")
threads = solver_options[option_set].get("threads", 4)
return threads
@ -105,7 +105,7 @@ def memory(w):
def input_custom_extra_functionality(w):
path = config["solving"]["options"].get("custom_extra_functionality", False)
path = config_provider("solving", "options", "custom_extra_functionality", default=False)
if path:
return os.path.join(os.path.dirname(workflow.snakefile), path)
return []
@ -129,12 +129,12 @@ def has_internet_access(url="www.zenodo.org") -> bool:
def input_eurostat(w):
# 2016 includes BA, 2017 does not
report_year = config["energy"]["eurostat_report_year"]
report_year = config_provider("energy", "eurostat_report_year")
return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition"
def solved_previous_horizon(wildcards):
planning_horizons = config["scenario"]["planning_horizons"]
planning_horizons = config_provider("scenario", "planning_horizons")
i = planning_horizons.index(int(wildcards.planning_horizons))
planning_horizon_p = str(planning_horizons[i - 1])
return (

68
rules/postprocess.smk
View File

@ -13,16 +13,15 @@ if config_provider("foresight") != "perfect":
params:
plotting=config_provider("plotting"),
input:
network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc",
regions_onshore=RESOURCES
+ "regions_onshore_elec_s{simpl}_{clusters}.geojson",
network=resources("networks/elec_s{simpl}_{clusters}.nc"),
regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
output:
map=RESULTS + "maps/power-network-s{simpl}-{clusters}.pdf",
threads: 1
resources:
mem_mb=4000,
benchmark:
BENCHMARKS + "plot_power_network_clustered/elec_s{simpl}_{clusters}"
benchmarks("plot_power_network_clustered/elec_s{simpl}_{clusters}")
conda:
"../envs/environment.yaml"
script:
@ -34,7 +33,7 @@ if config_provider("foresight") != "perfect":
input:
network=RESULTS
+ "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
regions=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
output:
map=RESULTS
+ "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf",
@ -42,15 +41,9 @@ if config_provider("foresight") != "perfect":
resources:
mem_mb=10000,
log:
(
LOGS
+ "plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"
),
logs("plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log")
benchmark:
(
BENCHMARKS
+ "plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
benchmarks("plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}")
conda:
"../envs/environment.yaml"
script:
@ -63,7 +56,7 @@ if config_provider("foresight") != "perfect":
input:
network=RESULTS
+ "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
regions=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
output:
map=RESULTS
+ "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf",
@ -71,15 +64,9 @@ if config_provider("foresight") != "perfect":
resources:
mem_mb=10000,
log:
(
LOGS
+ "plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"
),
logs("plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"),
benchmark:
(
BENCHMARKS
+ "plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
benchmarks("plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}")
conda:
"../envs/environment.yaml"
script:
@ -91,7 +78,7 @@ if config_provider("foresight") != "perfect":
input:
network=RESULTS
+ "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
regions=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
output:
map=RESULTS
+ "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-ch4_network_{planning_horizons}.pdf",
@ -99,15 +86,9 @@ if config_provider("foresight") != "perfect":
resources:
mem_mb=10000,
log:
(
LOGS
+ "plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"
),
logs("plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"),
benchmark:
(
BENCHMARKS
+ "plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
benchmarks("plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}")
conda:
"../envs/environment.yaml"
script:
@ -122,7 +103,7 @@ if config_provider("foresight") == "perfect":
input:
network=RESULTS
+ "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
regions=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"),
output:
**{
f"map_{year}": RESULTS
@ -134,8 +115,7 @@ if config_provider("foresight") == "perfect":
resources:
mem_mb=10000,
benchmark:
BENCHMARKS
+"postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_benchmark"
benchmarks("postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_benchmark")
conda:
"../envs/environment.yaml"
script:
@ -151,7 +131,7 @@ rule copy_config:
resources:
mem_mb=1000,
benchmark:
BENCHMARKS + "copy_config"
benchmarks("copy_config")
conda:
"../envs/environment.yaml"
script:
@ -162,7 +142,7 @@ rule make_summary:
params:
foresight=config_provider("foresight"),
costs=config_provider("costs"),
snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, # TODO: use config_provider
snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]},
scenario=config_provider("scenario"),
RDIR=RDIR,
input:
@ -177,9 +157,9 @@ rule make_summary:
run=config["run"]["name"],
),
costs=(
"data/costs_{}.csv".format(config["costs"]["year"])
"data/costs_{}.csv".format(config_provider("costs", "year"))
if config_provider("foresight") == "overnight"
else "data/costs_{}.csv".format(config["scenario"]["planning_horizons"][0])
else "data/costs_{}.csv".format(config_provider("scenario", "planning_horizons", 0))
),
ac_plot=expand(
RESULTS + "maps/power-network-s{simpl}-{clusters}.pdf",
@ -195,7 +175,7 @@ rule make_summary:
(
RESULTS
+ "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf"
if config["sector"]["H2_network"]
if config_provider("sector", "H2_network")
else []
),
**config["scenario"],
@ -205,7 +185,7 @@ rule make_summary:
(
RESULTS
+ "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-ch4_network_{planning_horizons}.pdf"
if config["sector"]["gas_network"]
if config_provider("sector", "gas_network")
else []
),
**config["scenario"],
@ -231,9 +211,9 @@ rule make_summary:
resources:
mem_mb=10000,
log:
LOGS + "make_summary.log",
logs("make_summary.log"),
benchmark:
BENCHMARKS + "make_summary"
benchmarks("make_summary")
conda:
"../envs/environment.yaml"
script:
@ -263,9 +243,9 @@ rule plot_summary:
resources:
mem_mb=10000,
log:
LOGS + "plot_summary.log",
logs("plot_summary.log"),
benchmark:
BENCHMARKS + "plot_summary"
benchmarks("plot_summary")
conda:
"../envs/environment.yaml"
script:

4
rules/retrieve.smk
View File

@ -50,7 +50,7 @@ if config["enable"].get("retrieve_irena"):
onwind="data/existing_infrastructure/onwind_capacity_IRENA.csv",
solar="data/existing_infrastructure/solar_capacity_IRENA.csv",
log:
LOGS + "retrieve_irena.log",
logs("retrieve_irena.log"),
resources:
mem_mb=1000,
retries: 2
@ -86,7 +86,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", T
input:
HTTP.remote(
"raw.githubusercontent.com/PyPSA/technology-data/{}/outputs/".format(
config["costs"]["version"]
config_provider("costs", "version")
)
+ "costs_{year}.csv",
keep_local=True,

18
rules/solve_electricity.smk
View File

@ -19,12 +19,11 @@ rule solve_network:
network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
log:
solver=normpath(
LOGS + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"
logs("solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log")
),
python=LOGS
+ "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log",
python=logs("solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log"),
benchmark:
BENCHMARKS + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}"
benchmarks("solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}")
threads: solver_threads
resources:
mem_mb=memory,
@ -46,16 +45,11 @@ rule solve_operations_network:
network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc",
log:
solver=normpath(
LOGS
+ "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"
logs("solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log")
),
python=LOGS
+ "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log",
python=logs("solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log"),
benchmark:
(
BENCHMARKS
+ "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}"
)
benchmarks("solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}")
threads: 4
resources:
mem_mb=(lambda w: 10000 + 372 * int(w.clusters)),

37
rules/solve_myopic.smk
View File

@ -36,13 +36,9 @@ rule add_existing_baseyear:
resources:
mem_mb=2000,
log:
LOGS
+ "add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
logs("add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"),
benchmark:
(
BENCHMARKS
+ "add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
benchmarks("add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}")
conda:
"../envs/environment.yaml"
script:
@ -56,16 +52,16 @@ rule add_brownfield:
"sector", "H2_retrofit_capacity_per_CH4"
),
threshold_capacity=config_provider("existing_capacities", " threshold_capacity"),
snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, # TODO: use config_provider
snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]},
carriers=config_provider("electricity", "renewable_carriers"),
input:
**{
f"profile_{tech}": RESOURCES + f"profile_{tech}.nc"
for tech in config["electricity"]["renewable_carriers"]
f"profile_{tech}": resources(f"profile_{tech}.nc")
for tech in config_provider("electricity", "renewable_carriers")
if tech != "hydro"
},
simplify_busmap=RESOURCES + "busmap_elec_s{simpl}.csv",
cluster_busmap=RESOURCES + "busmap_elec_s{simpl}_{clusters}.csv",
simplify_busmap=resources("busmap_elec_s{simpl}.csv"),
cluster_busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"),
network=RESULTS
+ "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
network_p=solved_previous_horizon, #solved network at previous time step
@ -79,13 +75,9 @@ rule add_brownfield:
resources:
mem_mb=10000,
log:
LOGS
+ "add_brownfield_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
logs("add_brownfield_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"),
benchmark:
(
BENCHMARKS
+ "add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
benchmarks("add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}")
conda:
"../envs/environment.yaml"
script:
@ -115,19 +107,14 @@ rule solve_sector_network_myopic:
shadow:
"shallow"
log:
solver=LOGS
+ "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log",
python=LOGS
+ "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log",
solver=logs("elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log"),
python=logs("elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log"),
threads: solver_threads
resources:
mem_mb=config_provider("solving", "mem"),
walltime=config_provider("solving", "walltime", default="12:00:00"),
benchmark:
(
BENCHMARKS
+ "solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
benchmarks("solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}")
conda:
"../envs/environment.yaml"
script:

5
rules/solve_overnight.smk
View File

@ -33,10 +33,7 @@ rule solve_sector_network:
mem_mb=config_provider("solving", "mem"),
walltime=config_provider("solving", "walltime", default="12:00:00"),
benchmark:
(
BENCHMARKS
+ "solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
benchmarks("solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}")
conda:
"../envs/environment.yaml"
script:

84
rules/solve_perfect.smk
View File

@ -3,22 +3,21 @@
# SPDX-License-Identifier: MIT
rule add_existing_baseyear:
params:
baseyear=config["scenario"]["planning_horizons"][0],
sector=config["sector"],
existing_capacities=config["existing_capacities"],
costs=config["costs"],
baseyear=config_provider("scenario", "planning_horizons", 0),
sector=config_provider("sector"),
existing_capacities=config_provider("existing_capacities"),
costs=config_provider("costs"),
input:
network=RESULTS
+ "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
powerplants=RESOURCES + "powerplants.csv",
busmap_s=RESOURCES + "busmap_elec_s{simpl}.csv",
busmap=RESOURCES + "busmap_elec_s{simpl}_{clusters}.csv",
clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv",
costs="data/costs_{}.csv".format(config["scenario"]["planning_horizons"][0]),
cop_soil_total=RESOURCES + "cop_soil_total_elec_s{simpl}_{clusters}.nc",
cop_air_total=RESOURCES + "cop_air_total_elec_s{simpl}_{clusters}.nc",
existing_heating_distribution=RESOURCES
+ "existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv",
powerplants=resources("powerplants.csv"),
busmap_s=resources("busmap_elec_s{simpl}.csv"),
busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"),
clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"),
costs="data/costs_{}.csv".format(config_provider("scenario", "planning_horizons", 0)),
cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"),
cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"),
existing_heating_distribution=resources("existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv"),
existing_heating="data/existing_infrastructure/existing_heating_raw.csv",
existing_solar="data/existing_infrastructure/solar_capacity_IRENA.csv",
existing_onwind="data/existing_infrastructure/onwind_capacity_IRENA.csv",
@ -27,18 +26,14 @@ rule add_existing_baseyear:
RESULTS
+ "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
wildcard_constraints:
planning_horizons=config["scenario"]["planning_horizons"][0], #only applies to baseyear
planning_horizons=config_provider("scenario", "planning_horizons", 0), #only applies to baseyear
threads: 1
resources:
mem_mb=2000,
log:
LOGS
+ "add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log",
logs("add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"),
benchmark:
(
BENCHMARKS
+ "add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
)
benchmarks("add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}")
conda:
"../envs/environment.yaml"
script:
@ -51,13 +46,13 @@ rule prepare_perfect_foresight:
f"network_{year}": RESULTS
+ "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_"
+ f"{year}.nc"
for year in config["scenario"]["planning_horizons"][1:]
for year in config_provider("scenario", "planning_horizons")[1:]
},
brownfield_network=lambda w: (
RESULTS
+ "prenetworks-brownfield/"
+ "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_"
+ "{}.nc".format(str(config["scenario"]["planning_horizons"][0]))
+ "{}.nc".format(str(config_provider("scenario", "planning_horizons", 0)))
),
output:
RESULTS
@ -66,13 +61,9 @@ rule prepare_perfect_foresight:
resources:
mem_mb=10000,
log:
LOGS
+ "prepare_perfect_foresight{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}.log",
logs("prepare_perfect_foresight{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}.log"),
benchmark:
(
BENCHMARKS
+ "prepare_perfect_foresight{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}"
)
benchmarks("prepare_perfect_foresight{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}")
conda:
"../envs/environment.yaml"
script:
@ -81,13 +72,11 @@ rule prepare_perfect_foresight:
rule solve_sector_network_perfect:
params:
solving=config["solving"],
foresight=config["foresight"],
sector=config["sector"],
planning_horizons=config["scenario"]["planning_horizons"],
co2_sequestration_potential=config["sector"].get(
"co2_sequestration_potential", 200
),
solving=config_provider("solving"),
foresight=config_provider("foresight"),
sector=config_provider("sector"),
planning_horizons=config_provider("scenario", "planning_horizons"),
co2_sequestration_potential=config_provider("sector", "co2_sequestration_potential", 200),
custom_extra_functionality=input_custom_extra_functionality,
input:
network=RESULTS
@ -99,7 +88,7 @@ rule solve_sector_network_perfect:
+ "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
threads: solver_threads
resources:
mem_mb=config["solving"]["mem"],
mem_mb=config_provider("solving", "mem"),
shadow:
"shallow"
log:
@ -110,10 +99,7 @@ rule solve_sector_network_perfect:
memory=RESULTS
+ "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_memory.log",
benchmark:
(
BENCHMARKS
+ "solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years}"
)
benchmarks("solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years}")
conda:
"../envs/environment.yaml"
script:
@ -124,13 +110,13 @@ rule make_summary_perfect:
input:
**{
f"networks_{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}": RESULTS
+ f"postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc"
for simpl in config["scenario"]["simpl"]
for clusters in config["scenario"]["clusters"]
for opts in config["scenario"]["opts"]
for sector_opts in config["scenario"]["sector_opts"]
for ll in config["scenario"]["ll"]
},
+ f"postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc"
for simpl in config_provider("scenario", "simpl")
for clusters in config_provider("scenario", "clusters")
for opts in config_provider("scenario", "opts")
for sector_opts in config_provider("scenario", "sector_opts")
for ll in config_provider("scenario", "ll")
},
costs="data/costs_2020.csv",
output:
nodal_costs=RESULTS + "csvs/nodal_costs.csv",
@ -153,9 +139,9 @@ rule make_summary_perfect:
resources:
mem_mb=10000,
log:
LOGS + "make_summary_perfect.log",
logs("make_summary_perfect.log"),
benchmark:
(BENCHMARKS + "make_summary_perfect")
benchmarks("make_summary_perfect")
conda:
"../envs/environment.yaml"
script:

14
rules/validate.smk
View File

@ -17,8 +17,8 @@ rule build_electricity_production:
The data is used for validation of the optimization results.
"""
params:
snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
countries=config["countries"],
snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]},
countries=config_provider("countries"),
output:
resources("historical_electricity_production.csv"),
log:
@ -35,8 +35,8 @@ rule build_cross_border_flows:
The data is used for validation of the optimization results.
"""
params:
snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
countries=config["countries"],
snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]},
countries=config_provider("countries"),
input:
network=resources("networks/base.nc"),
output:
@ -55,8 +55,8 @@ rule build_electricity_prices:
The data is used for validation of the optimization results.
"""
params:
snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
countries=config["countries"],
snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]},
countries=config_provider("countries"),
output:
resources("historical_electricity_prices.csv"),
log:
@ -85,7 +85,7 @@ rule plot_validation_electricity_production:
rule plot_validation_cross_border_flows:
params:
countries=config["countries"],
countries=config_provider("countries"),
input:
network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
cross_border_flows=resources("historical_cross_border_flows.csv"),