From a9dad3f34ece29af49481738b1d0b6ddd43d67d1 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 10 Feb 2024 18:09:23 +0100 Subject: [PATCH] add new resources(), logs(), benchmarks(), config_providers() --- Snakefile | 2 +- rules/build_electricity.smk | 63 +++++++++++++------------- rules/build_sector.smk | 90 ++++++++++++++++--------------------- rules/collect.smk | 6 --- rules/common.smk | 10 ++--- rules/postprocess.smk | 68 ++++++++++------------------ rules/retrieve.smk | 4 +- rules/solve_electricity.smk | 18 +++----- rules/solve_myopic.smk | 37 +++++---------- rules/solve_overnight.smk | 5 +-- rules/solve_perfect.smk | 84 +++++++++++++++------------------- rules/validate.smk | 14 +++--- 12 files changed, 163 insertions(+), 238 deletions(-) diff --git a/Snakefile b/Snakefile index e0949ed9..7df61162 100644 --- a/Snakefile +++ b/Snakefile @@ -91,7 +91,7 @@ if config["foresight"] == "perfect": rule all: input: - RESULTS + "graphs/costs.pdf", + expand(RESULTS + "graphs/costs.pdf", run=config["run"]["name"]), default_target: True diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 4c6650bc..951d3331 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -20,7 +20,7 @@ if config["enable"].get("prepare_links_p_nom", False): rule build_electricity_demand: params: - snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, # TODO: use config provider + snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]}, countries=config_provider("countries"), load=config_provider("load"), input: @@ -62,7 +62,7 @@ rule build_powerplants: rule base_network: params: countries=config_provider("countries"), - snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, # TODO: use config provider + snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]}, lines=config_provider("lines"), links=config_provider("links"), transformers=config_provider("transformers"), @@ -145,7 +145,7 @@ if config["enable"].get("build_cutout", False): rule build_cutout: params: - snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, # TODO: use config provider + snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]}, cutouts=config_provider("atlite", "cutouts"), input: regions_onshore=resources("regions_onshore.geojson"), @@ -170,7 +170,7 @@ if config["enable"].get("build_natura_raster", False): rule build_natura_raster: input: natura=ancient("data/bundle/natura/Natura2000_end2015.shp"), - cutouts=expand("cutouts/" + CDIR + "{cutouts}.nc", **config["atlite"]), + cutouts=expand("cutouts/" + CDIR + "{cutouts}.nc", **config_provider("atlite")), output: resources("natura.tiff"), resources: @@ -189,8 +189,8 @@ rule build_ship_raster: cutouts=expand( "cutouts/" + CDIR + "{cutout}.nc", cutout=[ - config["renewable"][k]["cutout"] - for k in config["electricity"]["renewable_carriers"] + config_provider("renewable", k, "cutout") + for k in config_provider("electricity", "renewable_carriers") ], ), output: @@ -214,30 +214,30 @@ rule determine_availability_matrix_MD_UA: wdpa_marine="data/WDPA_WDOECM_marine.gpkg", gebco=lambda w: ( "data/bundle/GEBCO_2014_2D.nc" - if "max_depth" in config["renewable"][w.technology].keys() + if config_provider("renewable", w.technology)(w).get("max_depth") else [] ), ship_density=lambda w: ( - RESOURCES + "shipdensity_raster.tif" - if "ship_threshold" in config["renewable"][w.technology].keys() + resources("shipdensity_raster.tif") + if "ship_threshold" in config_provider("renewable", w.technology)(w).keys() else [] ), - country_shapes=RESOURCES + "country_shapes.geojson", - offshore_shapes=RESOURCES + "offshore_shapes.geojson", + country_shapes=resources("country_shapes.geojson"), + offshore_shapes=resources("offshore_shapes.geojson"), regions=lambda w: ( - RESOURCES + "regions_onshore.geojson" + resources("regions_onshore.geojson") if w.technology in ("onwind", "solar") - else RESOURCES + "regions_offshore.geojson" + else resources("regions_offshore.geojson") ), cutout=lambda w: "cutouts/" + CDIR - + config["renewable"][w.technology]["cutout"] + + config_provider("renewable", w.technology, "cutout")(w) + ".nc", output: - availability_matrix=RESOURCES + "availability_matrix_MD-UA_{technology}.nc", - availability_map=RESOURCES + "availability_matrix_MD-UA_{technology}.png", + availability_matrix=resources("availability_matrix_MD-UA_{technology}.nc"), + availability_map=resources("availability_matrix_MD-UA_{technology}.png"), log: - LOGS + "determine_availability_matrix_MD_UA_{technology}.log", + logs("determine_availability_matrix_MD_UA_{technology}.log"), threads: ATLITE_NPROCESSES resources: mem_mb=ATLITE_NPROCESSES * 5000, @@ -250,8 +250,7 @@ rule determine_availability_matrix_MD_UA: # Optional input when having Ukraine (UA) or Moldova (MD) in the countries list if {"UA", "MD"}.intersection(set(config["countries"])): opt = { - "availability_matrix_MD_UA": RESOURCES - + "availability_matrix_MD-UA_{technology}.nc" + "availability_matrix_MD_UA": resources("availability_matrix_MD-UA_{technology}.nc") } else: opt = {} @@ -259,7 +258,7 @@ else: rule build_renewable_profiles: params: - snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, # TODO: use config provider + snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]}, renewable=config_provider("renewable"), input: **opt, @@ -272,7 +271,7 @@ rule build_renewable_profiles: ), luisa=lambda w: ( "data/LUISA_basemap_020321_50m.tif" - if config["renewable"][w.technology].get("luisa") + if config_provider("renewable", w.technology, "luisa")(w) else [] ), gebco=ancient( @@ -340,7 +339,7 @@ rule build_hydro_profile: input: country_shapes=resources("country_shapes.geojson"), eia_hydro_generation="data/eia_hydro_annual_generation.csv", - cutout=f"cutouts/" + CDIR + config["renewable"]["hydro"]["cutout"] + ".nc", + cutout=f"cutouts/" + CDIR + config_provider("renewable", "hydro", "cutout") + ".nc", output: resources("profile_hydro.nc"), log: @@ -357,12 +356,12 @@ if config["lines"]["dynamic_line_rating"]["activate"]: rule build_line_rating: params: - snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, + snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]}, input: base_network=resources("networks/base.nc"), cutout="cutouts/" + CDIR - + config["lines"]["dynamic_line_rating"]["cutout"] + + config_provider("lines", "dynamic_line_rating", "cutout") + ".nc", output: output=resources("networks/line_rating.nc"), @@ -391,19 +390,19 @@ rule add_electricity: input: **{ f"profile_{tech}": resources(f"profile_{tech}.nc") - for tech in config["electricity"]["renewable_carriers"] + for tech in config_provider("electricity", "renewable_carriers") }, **{ f"conventional_{carrier}_{attr}": fn for carrier, d in config.get("conventional", {None: {}}).items() - if carrier in config["electricity"]["conventional_carriers"] + if carrier in config_provider("electricity", "conventional_carriers") for attr, fn in d.items() if str(fn).startswith("data/") }, base_network=resources("networks/base.nc"), line_rating=( resources("networks/line_rating.nc") - if config["lines"]["dynamic_line_rating"]["activate"] + if config_provider("lines", "dynamic_line_rating", "activate") else resources("networks/base.nc") ), tech_costs=COSTS, @@ -414,7 +413,7 @@ rule add_electricity: unit_commitment="data/unit_commitment.csv", fuel_price=( resources("monthly_fuel_price.csv") - if config["conventional"]["dynamic_fuel_price"] + if config_provider("conventional", "dynamic_fuel_price") else [] ), load=resources("load.csv"), @@ -493,7 +492,7 @@ rule cluster_network: busmap=ancient(resources("busmap_elec_s{simpl}.csv")), custom_busmap=( "data/custom_busmap_elec_s{simpl}_{clusters}.csv" - if config["enable"].get("custom_busmap", False) + if config_provider("enable", "custom_busmap", default=False) else [] ), tech_costs=COSTS, @@ -542,10 +541,10 @@ rule add_extra_components: rule prepare_network: params: snapshots={ - "resolution": config["snapshots"].get("resolution", False), - "segmentation": config["snapshots"].get("segmentation", False), + "resolution": config_provider("snapshots", "resolution", default=False), + "segmentation": config_provider("snapshots", "segmentation", default=False), }, - # TODO: use config provider + links=config_provider("links"), lines=config_provider("lines"), co2base=config_provider("electricity", "co2base"), diff --git a/rules/build_sector.smk b/rules/build_sector.smk index aea16519..a26efaf4 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -7,7 +7,7 @@ rule build_population_layouts: input: nuts3_shapes=resources("nuts3_shapes.geojson"), urban_percent="data/urban_percent.csv", - cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", + cutout="cutouts/" + CDIR + config_provider("atlite", "default_cutout") + ".nc", output: pop_layout_total=resources("pop_layout_total.nc"), pop_layout_urban=resources("pop_layout_urban.nc"), @@ -31,7 +31,7 @@ rule build_clustered_population_layouts: pop_layout_urban=resources("pop_layout_urban.nc"), pop_layout_rural=resources("pop_layout_rural.nc"), regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), - cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", + cutout="cutouts/" + CDIR + config_provider("atlite", "default_cutout") + ".nc", output: clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), log: @@ -52,7 +52,7 @@ rule build_simplified_population_layouts: pop_layout_urban=resources("pop_layout_urban.nc"), pop_layout_rural=resources("pop_layout_rural.nc"), regions_onshore=resources("regions_onshore_elec_s{simpl}.geojson"), - cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", + cutout="cutouts/" + CDIR + config_provider("atlite", "default_cutout") + ".nc", output: clustered_pop_layout=resources("pop_layout_elec_s{simpl}.csv"), resources: @@ -126,11 +126,11 @@ rule cluster_gas_network: rule build_daily_heat_demand: params: - snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, # TODO: use config_provider + snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]}, input: pop_layout=resources("pop_layout_{scope}.nc"), regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), - cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", + cutout="cutouts/" + CDIR + config_provider("atlite", "default_cutout") + ".nc", output: heat_demand=resources("daily_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc"), resources: @@ -148,19 +148,19 @@ rule build_daily_heat_demand: rule build_hourly_heat_demand: params: - snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, + snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]}, input: heat_profile="data/heat_load_profile_BDEW.csv", - heat_demand=RESOURCES + "daily_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc", + heat_demand=resources("daily_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc"), output: - heat_demand=RESOURCES + "hourly_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc", + heat_demand=resources("hourly_heat_demand_{scope}_elec_s{simpl}_{clusters}.nc"), resources: mem_mb=2000, threads: 8 log: - LOGS + "build_hourly_heat_demand_{scope}_{simpl}_{clusters}.loc", + logs("build_hourly_heat_demand_{scope}_{simpl}_{clusters}.loc"), benchmark: - BENCHMARKS + "build_hourly_heat_demand/{scope}_s{simpl}_{clusters}" + benchmarks("build_hourly_heat_demand/{scope}_s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -169,11 +169,11 @@ rule build_hourly_heat_demand: rule build_temperature_profiles: params: - snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, # TODO: use config_provider + snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]}, input: pop_layout=resources("pop_layout_{scope}.nc"), regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), - cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", + cutout="cutouts/" + CDIR + config_provider("atlite", "default_cutout") + ".nc", output: temp_soil=resources("temp_soil_{scope}_elec_s{simpl}_{clusters}.nc"), temp_air=resources("temp_air_{scope}_elec_s{simpl}_{clusters}.nc"), @@ -221,12 +221,12 @@ rule build_cop_profiles: rule build_solar_thermal_profiles: params: - snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, # TODO use config_provider + snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]}, # TODO use config_provider solar_thermal=config_provider("solar_thermal"), input: pop_layout=resources("pop_layout_{scope}.nc"), regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), - cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", + cutout="cutouts/" + CDIR + config_provider("atlite", "default_cutout") + ".nc", output: solar_thermal=resources("solar_thermal_{scope}_elec_s{simpl}_{clusters}.nc"), resources: @@ -711,7 +711,7 @@ rule build_shipping_demand: rule build_transport_demand: params: - snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, # TODO: use config_provider + snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]}, sector=config_provider("sector"), input: clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), @@ -740,18 +740,17 @@ rule build_transport_demand: rule build_district_heat_share: params: - sector=config["sector"], + sector=config_provider("sector"), input: - district_heat_share=RESOURCES + "district_heat_share.csv", - clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", + district_heat_share=resources("district_heat_share.csv"), + clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), output: - district_heat_share=RESOURCES - + "district_heat_share_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + district_heat_share=resources("district_heat_share_elec_s{simpl}_{clusters}_{planning_horizons}.csv"), threads: 1 resources: mem_mb=1000, log: - LOGS + "build_district_heat_share_s{simpl}_{clusters}_{planning_horizons}.log", + logs("build_district_heat_share_s{simpl}_{clusters}_{planning_horizons}.log"), conda: "../envs/environment.yaml" script: @@ -760,32 +759,25 @@ rule build_district_heat_share: rule build_existing_heating_distribution: params: - baseyear=config["scenario"]["planning_horizons"][0], - sector=config["sector"], - existing_capacities=config["existing_capacities"], + baseyear=config_provider("scenario", "planning_horizons", 0), + sector=config_provider("sector"), + existing_capacities=config_provider("existing_capacities"), input: existing_heating="data/existing_infrastructure/existing_heating_raw.csv", - clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", - clustered_pop_energy_layout=RESOURCES - + "pop_weighted_energy_totals_s{simpl}_{clusters}.csv", - district_heat_share=RESOURCES - + "district_heat_share_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), + clustered_pop_energy_layout=resources("pop_weighted_energy_totals_s{simpl}_{clusters}.csv"), + district_heat_share=resources("district_heat_share_elec_s{simpl}_{clusters}_{planning_horizons}.csv"), output: - existing_heating_distribution=RESOURCES - + "existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + existing_heating_distribution=resources("existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv"), wildcard_constraints: - planning_horizons=config["scenario"]["planning_horizons"][0], #only applies to baseyear + planning_horizons=config_provider("scenario", "planning_horizons", 0), #only applies to baseyear threads: 1 resources: mem_mb=2000, log: - LOGS - + "build_existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.log", + logs("build_existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.log"), benchmark: - ( - BENCHMARKS - + "build_existing_heating_distribution/elec_s{simpl}_{clusters}_{planning_horizons}" - ) + benchmarks("build_existing_heating_distribution/elec_s{simpl}_{clusters}_{planning_horizons}") conda: "../envs/environment.yaml" script: @@ -831,16 +823,16 @@ rule prepare_sector_network: biomass_potentials=( resources( "biomass_potentials_s{simpl}_{clusters}_" - + "{}.csv".format(config["biomass"]["year"]) + + "{}.csv".format(config_provider("biomass", "year")) ) - if config["foresight"] == "overnight" + if config_provider("foresight") == "overnight" else resources( "biomass_potentials_s{simpl}_{clusters}_{planning_horizons}.csv" ) ), costs=( - "data/costs_{}.csv".format(config["costs"]["year"]) - if config["foresight"] == "overnight" + "data/costs_{}.csv".format(config_provider("costs", "year")) + if config_provider("foresight") == "overnight" else "data/costs_{planning_horizons}.csv" ), profile_offwind_ac=resources("profile_offwind-ac.nc"), @@ -873,17 +865,17 @@ rule prepare_sector_network: cop_air_urban=resources("cop_air_urban_elec_s{simpl}_{clusters}.nc"), solar_thermal_total=( resources("solar_thermal_total_elec_s{simpl}_{clusters}.nc") - if config["sector"]["solar_thermal"] + if config_provider("sector", "solar_thermal") else [] ), solar_thermal_urban=( resources("solar_thermal_urban_elec_s{simpl}_{clusters}.nc") - if config["sector"]["solar_thermal"] + if config_provider("sector", "solar_thermal") else [] ), solar_thermal_rural=( resources("solar_thermal_rural_elec_s{simpl}_{clusters}.nc") - if config["sector"]["solar_thermal"] + if config_provider("sector", "solar_thermal") else [] ), output: @@ -893,13 +885,9 @@ rule prepare_sector_network: resources: mem_mb=2000, log: - LOGS - + "prepare_sector_network_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", + logs("prepare_sector_network_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"), benchmark: - ( - BENCHMARKS - + "prepare_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" - ) + benchmarks("prepare_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}") conda: "../envs/environment.yaml" script: diff --git a/rules/collect.smk b/rules/collect.smk index 8a451d7a..9be12f25 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -13,12 +13,6 @@ localrules: solve_sector_networks, -rule all: - input: - expand(RESULTS + "graphs/costs.pdf", run=config["run"]["name"]), - default_target: True - - rule cluster_networks: input: expand( diff --git a/rules/common.smk b/rules/common.smk index b6c0f734..af991be7 100644 --- a/rules/common.smk +++ b/rules/common.smk @@ -78,8 +78,8 @@ def config_provider(*keys, default=None): def solver_threads(w): - solver_options = config["solving"]["solver_options"] - option_set = config["solving"]["solver"]["options"] + solver_options = config_provider("solving", "solver_options") + option_set = config_provider("solving", "solver", "options") threads = solver_options[option_set].get("threads", 4) return threads @@ -105,7 +105,7 @@ def memory(w): def input_custom_extra_functionality(w): - path = config["solving"]["options"].get("custom_extra_functionality", False) + path = config_provider("solving", "options", "custom_extra_functionality", default=False) if path: return os.path.join(os.path.dirname(workflow.snakefile), path) return [] @@ -129,12 +129,12 @@ def has_internet_access(url="www.zenodo.org") -> bool: def input_eurostat(w): # 2016 includes BA, 2017 does not - report_year = config["energy"]["eurostat_report_year"] + report_year = config_provider("energy", "eurostat_report_year") return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition" def solved_previous_horizon(wildcards): - planning_horizons = config["scenario"]["planning_horizons"] + planning_horizons = config_provider("scenario", "planning_horizons") i = planning_horizons.index(int(wildcards.planning_horizons)) planning_horizon_p = str(planning_horizons[i - 1]) return ( diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 7e269688..98399cde 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -13,16 +13,15 @@ if config_provider("foresight") != "perfect": params: plotting=config_provider("plotting"), input: - network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", - regions_onshore=RESOURCES - + "regions_onshore_elec_s{simpl}_{clusters}.geojson", + network=resources("networks/elec_s{simpl}_{clusters}.nc"), + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), output: map=RESULTS + "maps/power-network-s{simpl}-{clusters}.pdf", threads: 1 resources: mem_mb=4000, benchmark: - BENCHMARKS + "plot_power_network_clustered/elec_s{simpl}_{clusters}" + benchmarks("plot_power_network_clustered/elec_s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -34,7 +33,7 @@ if config_provider("foresight") != "perfect": input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", + regions=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), output: map=RESULTS + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", @@ -42,15 +41,9 @@ if config_provider("foresight") != "perfect": resources: mem_mb=10000, log: - ( - LOGS - + "plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" - ), + logs("plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log") benchmark: - ( - BENCHMARKS - + "plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" - ) + benchmarks("plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}") conda: "../envs/environment.yaml" script: @@ -63,7 +56,7 @@ if config_provider("foresight") != "perfect": input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", + regions=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), output: map=RESULTS + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf", @@ -71,15 +64,9 @@ if config_provider("foresight") != "perfect": resources: mem_mb=10000, log: - ( - LOGS - + "plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" - ), + logs("plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"), benchmark: - ( - BENCHMARKS - + "plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" - ) + benchmarks("plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}") conda: "../envs/environment.yaml" script: @@ -91,7 +78,7 @@ if config_provider("foresight") != "perfect": input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", + regions=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), output: map=RESULTS + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-ch4_network_{planning_horizons}.pdf", @@ -99,15 +86,9 @@ if config_provider("foresight") != "perfect": resources: mem_mb=10000, log: - ( - LOGS - + "plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" - ), + logs("plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"), benchmark: - ( - BENCHMARKS - + "plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" - ) + benchmarks("plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}") conda: "../envs/environment.yaml" script: @@ -122,7 +103,7 @@ if config_provider("foresight") == "perfect": input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", - regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", + regions=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), output: **{ f"map_{year}": RESULTS @@ -134,8 +115,7 @@ if config_provider("foresight") == "perfect": resources: mem_mb=10000, benchmark: - BENCHMARKS - +"postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_benchmark" + benchmarks("postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_benchmark") conda: "../envs/environment.yaml" script: @@ -151,7 +131,7 @@ rule copy_config: resources: mem_mb=1000, benchmark: - BENCHMARKS + "copy_config" + benchmarks("copy_config") conda: "../envs/environment.yaml" script: @@ -162,7 +142,7 @@ rule make_summary: params: foresight=config_provider("foresight"), costs=config_provider("costs"), - snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, # TODO: use config_provider + snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]}, scenario=config_provider("scenario"), RDIR=RDIR, input: @@ -177,9 +157,9 @@ rule make_summary: run=config["run"]["name"], ), costs=( - "data/costs_{}.csv".format(config["costs"]["year"]) + "data/costs_{}.csv".format(config_provider("costs", "year")) if config_provider("foresight") == "overnight" - else "data/costs_{}.csv".format(config["scenario"]["planning_horizons"][0]) + else "data/costs_{}.csv".format(config_provider("scenario", "planning_horizons", 0)) ), ac_plot=expand( RESULTS + "maps/power-network-s{simpl}-{clusters}.pdf", @@ -195,7 +175,7 @@ rule make_summary: ( RESULTS + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf" - if config["sector"]["H2_network"] + if config_provider("sector", "H2_network") else [] ), **config["scenario"], @@ -205,7 +185,7 @@ rule make_summary: ( RESULTS + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-ch4_network_{planning_horizons}.pdf" - if config["sector"]["gas_network"] + if config_provider("sector", "gas_network") else [] ), **config["scenario"], @@ -231,9 +211,9 @@ rule make_summary: resources: mem_mb=10000, log: - LOGS + "make_summary.log", + logs("make_summary.log"), benchmark: - BENCHMARKS + "make_summary" + benchmarks("make_summary") conda: "../envs/environment.yaml" script: @@ -263,9 +243,9 @@ rule plot_summary: resources: mem_mb=10000, log: - LOGS + "plot_summary.log", + logs("plot_summary.log"), benchmark: - BENCHMARKS + "plot_summary" + benchmarks("plot_summary") conda: "../envs/environment.yaml" script: diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 46741830..a0647e57 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -50,7 +50,7 @@ if config["enable"].get("retrieve_irena"): onwind="data/existing_infrastructure/onwind_capacity_IRENA.csv", solar="data/existing_infrastructure/solar_capacity_IRENA.csv", log: - LOGS + "retrieve_irena.log", + logs("retrieve_irena.log"), resources: mem_mb=1000, retries: 2 @@ -86,7 +86,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", T input: HTTP.remote( "raw.githubusercontent.com/PyPSA/technology-data/{}/outputs/".format( - config["costs"]["version"] + config_provider("costs", "version") ) + "costs_{year}.csv", keep_local=True, diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index fc8e8cea..4ff94bf2 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -19,12 +19,11 @@ rule solve_network: network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", log: solver=normpath( - LOGS + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log" + logs("solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log") ), - python=LOGS - + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", + python=logs("solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log"), benchmark: - BENCHMARKS + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" + benchmarks("solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}") threads: solver_threads resources: mem_mb=memory, @@ -46,16 +45,11 @@ rule solve_operations_network: network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc", log: solver=normpath( - LOGS - + "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log" + logs("solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log") ), - python=LOGS - + "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", + python=logs("solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log"), benchmark: - ( - BENCHMARKS - + "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" - ) + benchmarks("solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}") threads: 4 resources: mem_mb=(lambda w: 10000 + 372 * int(w.clusters)), diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 7035f1c1..260837b7 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -36,13 +36,9 @@ rule add_existing_baseyear: resources: mem_mb=2000, log: - LOGS - + "add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", + logs("add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"), benchmark: - ( - BENCHMARKS - + "add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" - ) + benchmarks("add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}") conda: "../envs/environment.yaml" script: @@ -56,16 +52,16 @@ rule add_brownfield: "sector", "H2_retrofit_capacity_per_CH4" ), threshold_capacity=config_provider("existing_capacities", " threshold_capacity"), - snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, # TODO: use config_provider + snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]}, carriers=config_provider("electricity", "renewable_carriers"), input: **{ - f"profile_{tech}": RESOURCES + f"profile_{tech}.nc" - for tech in config["electricity"]["renewable_carriers"] + f"profile_{tech}": resources(f"profile_{tech}.nc") + for tech in config_provider("electricity", "renewable_carriers") if tech != "hydro" }, - simplify_busmap=RESOURCES + "busmap_elec_s{simpl}.csv", - cluster_busmap=RESOURCES + "busmap_elec_s{simpl}_{clusters}.csv", + simplify_busmap=resources("busmap_elec_s{simpl}.csv"), + cluster_busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"), network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", network_p=solved_previous_horizon, #solved network at previous time step @@ -79,13 +75,9 @@ rule add_brownfield: resources: mem_mb=10000, log: - LOGS - + "add_brownfield_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", + logs("add_brownfield_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"), benchmark: - ( - BENCHMARKS - + "add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" - ) + benchmarks("add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}") conda: "../envs/environment.yaml" script: @@ -115,19 +107,14 @@ rule solve_sector_network_myopic: shadow: "shallow" log: - solver=LOGS - + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log", - python=LOGS - + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log", + solver=logs("elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log"), + python=logs("elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log"), threads: solver_threads resources: mem_mb=config_provider("solving", "mem"), walltime=config_provider("solving", "walltime", default="12:00:00"), benchmark: - ( - BENCHMARKS - + "solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" - ) + benchmarks("solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}") conda: "../envs/environment.yaml" script: diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index 76621012..7811efe3 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -33,10 +33,7 @@ rule solve_sector_network: mem_mb=config_provider("solving", "mem"), walltime=config_provider("solving", "walltime", default="12:00:00"), benchmark: - ( - BENCHMARKS - + "solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" - ) + benchmarks("solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}") conda: "../envs/environment.yaml" script: diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 9e164a16..85f87d9b 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -3,22 +3,21 @@ # SPDX-License-Identifier: MIT rule add_existing_baseyear: params: - baseyear=config["scenario"]["planning_horizons"][0], - sector=config["sector"], - existing_capacities=config["existing_capacities"], - costs=config["costs"], + baseyear=config_provider("scenario", "planning_horizons", 0), + sector=config_provider("sector"), + existing_capacities=config_provider("existing_capacities"), + costs=config_provider("costs"), input: network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - powerplants=RESOURCES + "powerplants.csv", - busmap_s=RESOURCES + "busmap_elec_s{simpl}.csv", - busmap=RESOURCES + "busmap_elec_s{simpl}_{clusters}.csv", - clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", - costs="data/costs_{}.csv".format(config["scenario"]["planning_horizons"][0]), - cop_soil_total=RESOURCES + "cop_soil_total_elec_s{simpl}_{clusters}.nc", - cop_air_total=RESOURCES + "cop_air_total_elec_s{simpl}_{clusters}.nc", - existing_heating_distribution=RESOURCES - + "existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + powerplants=resources("powerplants.csv"), + busmap_s=resources("busmap_elec_s{simpl}.csv"), + busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"), + clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), + costs="data/costs_{}.csv".format(config_provider("scenario", "planning_horizons", 0)), + cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"), + cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"), + existing_heating_distribution=resources("existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv"), existing_heating="data/existing_infrastructure/existing_heating_raw.csv", existing_solar="data/existing_infrastructure/solar_capacity_IRENA.csv", existing_onwind="data/existing_infrastructure/onwind_capacity_IRENA.csv", @@ -27,18 +26,14 @@ rule add_existing_baseyear: RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", wildcard_constraints: - planning_horizons=config["scenario"]["planning_horizons"][0], #only applies to baseyear + planning_horizons=config_provider("scenario", "planning_horizons", 0), #only applies to baseyear threads: 1 resources: mem_mb=2000, log: - LOGS - + "add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", + logs("add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log"), benchmark: - ( - BENCHMARKS - + "add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" - ) + benchmarks("add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}") conda: "../envs/environment.yaml" script: @@ -51,13 +46,13 @@ rule prepare_perfect_foresight: f"network_{year}": RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_" + f"{year}.nc" - for year in config["scenario"]["planning_horizons"][1:] + for year in config_provider("scenario", "planning_horizons")[1:] }, brownfield_network=lambda w: ( RESULTS + "prenetworks-brownfield/" + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_" - + "{}.nc".format(str(config["scenario"]["planning_horizons"][0])) + + "{}.nc".format(str(config_provider("scenario", "planning_horizons", 0))) ), output: RESULTS @@ -66,13 +61,9 @@ rule prepare_perfect_foresight: resources: mem_mb=10000, log: - LOGS - + "prepare_perfect_foresight{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}.log", + logs("prepare_perfect_foresight{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}.log"), benchmark: - ( - BENCHMARKS - + "prepare_perfect_foresight{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}" - ) + benchmarks("prepare_perfect_foresight{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}") conda: "../envs/environment.yaml" script: @@ -81,13 +72,11 @@ rule prepare_perfect_foresight: rule solve_sector_network_perfect: params: - solving=config["solving"], - foresight=config["foresight"], - sector=config["sector"], - planning_horizons=config["scenario"]["planning_horizons"], - co2_sequestration_potential=config["sector"].get( - "co2_sequestration_potential", 200 - ), + solving=config_provider("solving"), + foresight=config_provider("foresight"), + sector=config_provider("sector"), + planning_horizons=config_provider("scenario", "planning_horizons"), + co2_sequestration_potential=config_provider("sector", "co2_sequestration_potential", 200), custom_extra_functionality=input_custom_extra_functionality, input: network=RESULTS @@ -99,7 +88,7 @@ rule solve_sector_network_perfect: + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", threads: solver_threads resources: - mem_mb=config["solving"]["mem"], + mem_mb=config_provider("solving", "mem"), shadow: "shallow" log: @@ -110,10 +99,7 @@ rule solve_sector_network_perfect: memory=RESULTS + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_memory.log", benchmark: - ( - BENCHMARKS - + "solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years}" - ) + benchmarks("solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years}") conda: "../envs/environment.yaml" script: @@ -124,13 +110,13 @@ rule make_summary_perfect: input: **{ f"networks_{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}": RESULTS - + f"postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc" - for simpl in config["scenario"]["simpl"] - for clusters in config["scenario"]["clusters"] - for opts in config["scenario"]["opts"] - for sector_opts in config["scenario"]["sector_opts"] - for ll in config["scenario"]["ll"] - }, + + f"postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc" + for simpl in config_provider("scenario", "simpl") + for clusters in config_provider("scenario", "clusters") + for opts in config_provider("scenario", "opts") + for sector_opts in config_provider("scenario", "sector_opts") + for ll in config_provider("scenario", "ll") + }, costs="data/costs_2020.csv", output: nodal_costs=RESULTS + "csvs/nodal_costs.csv", @@ -153,9 +139,9 @@ rule make_summary_perfect: resources: mem_mb=10000, log: - LOGS + "make_summary_perfect.log", + logs("make_summary_perfect.log"), benchmark: - (BENCHMARKS + "make_summary_perfect") + benchmarks("make_summary_perfect") conda: "../envs/environment.yaml" script: diff --git a/rules/validate.smk b/rules/validate.smk index fefb6ba6..f8ebea5d 100644 --- a/rules/validate.smk +++ b/rules/validate.smk @@ -17,8 +17,8 @@ rule build_electricity_production: The data is used for validation of the optimization results. """ params: - snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, - countries=config["countries"], + snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]}, + countries=config_provider("countries"), output: resources("historical_electricity_production.csv"), log: @@ -35,8 +35,8 @@ rule build_cross_border_flows: The data is used for validation of the optimization results. """ params: - snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, - countries=config["countries"], + snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]}, + countries=config_provider("countries"), input: network=resources("networks/base.nc"), output: @@ -55,8 +55,8 @@ rule build_electricity_prices: The data is used for validation of the optimization results. """ params: - snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, - countries=config["countries"], + snapshots={k: config_provider("snapshots", k) for k in ["start", "end", "inclusive"]}, + countries=config_provider("countries"), output: resources("historical_electricity_prices.csv"), log: @@ -85,7 +85,7 @@ rule plot_validation_electricity_production: rule plot_validation_cross_border_flows: params: - countries=config["countries"], + countries=config_provider("countries"), input: network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", cross_border_flows=resources("historical_cross_border_flows.csv"),