diff --git a/Snakefile b/Snakefile index 79adc377..3d0c30bd 100644 --- a/Snakefile +++ b/Snakefile @@ -13,9 +13,14 @@ if not exists("config.yaml"): configfile: "config.yaml" -COSTS="resources/costs.csv" +run = config.get("run", {}) +RDIR = run.get("name", "") + "/" if run.get("name", "") else "" +CDIR = "" if run.get("dedicated_cutouts", True) else RDIR + +COSTS = "resources/" + RDIR + "costs.csv" ATLITE_NPROCESSES = config['atlite'].get('nprocesses', 4) + wildcard_constraints: simpl="[a-zA-Z0-9]*|all", clusters="[0-9]+m?|all", @@ -24,25 +29,25 @@ wildcard_constraints: rule cluster_all_networks: - input: expand("networks/elec_s{simpl}_{clusters}.nc", **config['scenario']) + input: expand("networks/" + RDIR + "elec_s{simpl}_{clusters}.nc", **config['scenario']) rule extra_components_all_networks: - input: expand("networks/elec_s{simpl}_{clusters}_ec.nc", **config['scenario']) + input: expand("networks/" + RDIR + "elec_s{simpl}_{clusters}_ec.nc", **config['scenario']) rule prepare_all_networks: - input: expand("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) + input: expand("networks/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) rule solve_all_networks: - input: expand("results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) + input: expand("results/networks/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) if config['enable'].get('prepare_links_p_nom', False): rule prepare_links_p_nom: output: 'data/links_p_nom.csv' - log: 'logs/prepare_links_p_nom.log' + log: "logs/" + RDIR + "prepare_links_p_nom.log" threads: 1 resources: mem_mb=500 script: 'scripts/prepare_links_p_nom.py' @@ -62,7 +67,7 @@ if not config.get('tutorial', False): if config['enable'].get('retrieve_databundle', True): rule retrieve_databundle: output: expand('data/bundle/{file}', file=datafiles) - log: "logs/retrieve_databundle.log" + log: "logs/" + RDIR + "retrieve_databundle.log" resources: mem_mb=1000 script: 'scripts/retrieve_databundle.py' @@ -76,17 +81,17 @@ rule retrieve_load_data: rule build_load_data: input: "data/load_raw.csv" - output: "resources/load.csv" - log: "logs/build_load_data.log" + output: "resources/" + RDIR + "load.csv" + log: "logs/" + RDIR + "build_load_data.log" resources: mem_mb=5000 script: 'scripts/build_load_data.py' rule build_powerplants: input: - base_network="networks/base.nc", + base_network="networks/" + RDIR + "base.nc", custom_powerplants="data/custom_powerplants.csv" - output: "resources/powerplants.csv" - log: "logs/build_powerplants.log" + output: "resources/" + RDIR + "powerplants.csv" + log: "logs/" + RDIR + "build_powerplants.log" threads: 1 resources: mem_mb=5000 script: "scripts/build_powerplants.py" @@ -102,12 +107,12 @@ rule base_network: parameter_corrections='data/parameter_corrections.yaml', links_p_nom='data/links_p_nom.csv', links_tyndp='data/links_tyndp.csv', - country_shapes='resources/country_shapes.geojson', - offshore_shapes='resources/offshore_shapes.geojson', - europe_shape='resources/europe_shape.geojson' - output: "networks/base.nc" - log: "logs/base_network.log" - benchmark: "benchmarks/base_network" + country_shapes="resources/" + RDIR + "country_shapes.geojson", + offshore_shapes="resources/" + RDIR + "offshore_shapes.geojson", + europe_shape="resources/" + RDIR + "europe_shape.geojson" + output: "networks/" + RDIR + "base.nc" + log: "logs/" + RDIR + "base_network.log" + benchmark: "benchmarks/" + RDIR + "base_network" threads: 1 resources: mem_mb=500 script: "scripts/base_network.py" @@ -123,11 +128,11 @@ rule build_shapes: ch_cantons='data/bundle/ch_cantons.csv', ch_popgdp='data/bundle/je-e-21.03.02.xls' output: - country_shapes='resources/country_shapes.geojson', - offshore_shapes='resources/offshore_shapes.geojson', - europe_shape='resources/europe_shape.geojson', - nuts3_shapes='resources/nuts3_shapes.geojson' - log: "logs/build_shapes.log" + country_shapes="resources/" + RDIR + "country_shapes.geojson", + offshore_shapes="resources/" + RDIR + "offshore_shapes.geojson", + europe_shape="resources/" + RDIR + "europe_shape.geojson", + nuts3_shapes="resources/" + RDIR + "nuts3_shapes.geojson" + log: "logs/" + RDIR + "build_shapes.log" threads: 1 resources: mem_mb=500 script: "scripts/build_shapes.py" @@ -135,13 +140,13 @@ rule build_shapes: rule build_bus_regions: input: - country_shapes='resources/country_shapes.geojson', - offshore_shapes='resources/offshore_shapes.geojson', - base_network="networks/base.nc" + country_shapes="resources/" + RDIR + "country_shapes.geojson", + offshore_shapes="resources/" + RDIR + "offshore_shapes.geojson", + base_network="networks/" + RDIR + "base.nc" output: - regions_onshore="resources/regions_onshore.geojson", - regions_offshore="resources/regions_offshore.geojson" - log: "logs/build_bus_regions.log" + regions_onshore="resources/" + RDIR + "regions_onshore.geojson", + regions_offshore="resources/" + RDIR + "regions_offshore.geojson" + log: "logs/" + RDIR + "build_bus_regions.log" threads: 1 resources: mem_mb=1000 script: "scripts/build_bus_regions.py" @@ -149,11 +154,11 @@ rule build_bus_regions: if config['enable'].get('build_cutout', False): rule build_cutout: input: - regions_onshore="resources/regions_onshore.geojson", - regions_offshore="resources/regions_offshore.geojson" - output: "cutouts/{cutout}.nc" - log: "logs/build_cutout/{cutout}.log" - benchmark: "benchmarks/build_cutout_{cutout}" + regions_onshore="resources/" + RDIR + "regions_onshore.geojson", + regions_offshore="resources/" + RDIR + "regions_offshore.geojson" + output: "cutouts/" + CDIR + "{cutout}.nc" + log: "logs/" + CDIR + "build_cutout/{cutout}.log" + benchmark: "benchmarks/" + CDIR + "build_cutout_{cutout}" threads: ATLITE_NPROCESSES resources: mem_mb=ATLITE_NPROCESSES * 1000 script: "scripts/build_cutout.py" @@ -162,8 +167,8 @@ if config['enable'].get('build_cutout', False): if config['enable'].get('retrieve_cutout', True): rule retrieve_cutout: input: HTTP.remote("zenodo.org/record/6382570/files/{cutout}.nc", keep_local=True, static=True) - output: "cutouts/{cutout}.nc" - log: "logs/retrieve_cutout_{cutout}.log" + output: "cutouts/" + CDIR + "{cutout}.nc" + log: "logs/" + CDIR + "retrieve_cutout_{cutout}.log" resources: mem_mb=5000 run: move(input[0], output[0]) @@ -171,7 +176,7 @@ if config['enable'].get('retrieve_cost_data', True): rule retrieve_cost_data: input: HTTP.remote(f"raw.githubusercontent.com/PyPSA/technology-data/{config['costs']['version']}/outputs/costs_{config['costs']['year']}.csv", keep_local=True) output: COSTS - log: "logs/retrieve_cost_data.log" + log: "logs/" + RDIR + "retrieve_cost_data.log" resources: mem_mb=5000 run: move(input[0], output[0]) @@ -179,17 +184,17 @@ if config['enable'].get('build_natura_raster', False): rule build_natura_raster: input: natura="data/bundle/natura/Natura2000_end2015.shp", - cutouts=expand("cutouts/{cutouts}.nc", **config['atlite']) - output: "resources/natura.tiff" + cutouts=expand("cutouts/" + CDIR + "{cutouts}.nc", **config['atlite']) + output: "resources/" + RDIR + "natura.tiff" resources: mem_mb=5000 - log: "logs/build_natura_raster.log" + log: "logs/" + RDIR + "build_natura_raster.log" script: "scripts/build_natura_raster.py" if config['enable'].get('retrieve_natura_raster', True): rule retrieve_natura_raster: input: HTTP.remote("zenodo.org/record/4706686/files/natura.tiff", keep_local=True, static=True) - output: "resources/natura.tiff" + output: "resources/" + RDIR + "natura.tiff" resources: mem_mb=5000 run: move(input[0], output[0]) @@ -204,36 +209,36 @@ rule retrieve_ship_raster: rule build_ship_raster: input: ship_density="data/shipdensity_global.zip", - cutouts=expand("cutouts/{cutouts}.nc", **config['atlite']) - output: "resources/shipdensity_raster.nc" - log: "logs/build_ship_raster.log" + cutouts=expand("cutouts/" + CDIR + "{cutouts}.nc", **config['atlite']) + output: "resources/" + RDIR + "shipdensity_raster.nc" + log: "logs/" + RDIR + "build_ship_raster.log" resources: mem_mb=5000 - benchmark: "benchmarks/build_ship_raster" + benchmark: "benchmarks/" + RDIR + "build_ship_raster" script: "scripts/build_ship_raster.py" rule build_renewable_profiles: input: - base_network="networks/base.nc", + base_network="networks/" + RDIR + "base.nc", corine="data/bundle/corine/g250_clc06_V18_5.tif", - natura=lambda w: ("resources/natura.tiff" + natura=lambda w: ("resources/" + RDIR + "natura.tiff" if config["renewable"][w.technology]["natura"] else []), gebco=lambda w: ("data/bundle/GEBCO_2014_2D.nc" if "max_depth" in config["renewable"][w.technology].keys() else []), - ship_density= lambda w: ("resources/shipdensity_raster.nc" + ship_density= lambda w: ("resources/" + RDIR + "shipdensity_raster.nc" if "ship_threshold" in config["renewable"][w.technology].keys() else []), - country_shapes='resources/country_shapes.geojson', - offshore_shapes='resources/offshore_shapes.geojson', - regions=lambda w: ("resources/regions_onshore.geojson" + country_shapes="resources/" + RDIR + "country_shapes.geojson", + offshore_shapes="resources/" + RDIR + "offshore_shapes.geojson", + regions=lambda w: ("resources/" + RDIR + "regions_onshore.geojson" if w.technology in ('onwind', 'solar') - else "resources/regions_offshore.geojson"), - cutout=lambda w: "cutouts/" + config["renewable"][w.technology]['cutout'] + ".nc" - output: profile="resources/profile_{technology}.nc", - log: "logs/build_renewable_profile_{technology}.log" - benchmark: "benchmarks/build_renewable_profiles_{technology}" + else "resources/" + RDIR + "regions_offshore.geojson"), + cutout=lambda w: "cutouts/" + CDIR + config["renewable"][w.technology]['cutout'] + ".nc" + output: profile="resources/" + RDIR + "profile_{technology}.nc", + log: "logs/" + RDIR + "build_renewable_profile_{technology}.log" + benchmark: "benchmarks/" + RDIR + "build_renewable_profiles_{technology}" threads: ATLITE_NPROCESSES resources: mem_mb=ATLITE_NPROCESSES * 5000 wildcard_constraints: technology="(?!hydro).*" # Any technology other than hydro @@ -242,31 +247,33 @@ rule build_renewable_profiles: rule build_hydro_profile: input: - country_shapes='resources/country_shapes.geojson', + country_shapes="resources/" + RDIR + "country_shapes.geojson", eia_hydro_generation='data/eia_hydro_annual_generation.csv', - cutout=f"cutouts/{config['renewable']['hydro']['cutout']}.nc" if "hydro" in config["renewable"] else "config['renewable']['hydro']['cutout'] not configured", - output: 'resources/profile_hydro.nc' - log: "logs/build_hydro_profile.log" + cutout=f"cutouts/" + CDIR + "{config['renewable']['hydro']['cutout']}.nc" if "hydro" in config["renewable"] else "config['renewable']['hydro']['cutout'] not configured", + output: "resources/" + RDIR + "profile_hydro.nc" + log: "logs/" + RDIR + "build_hydro_profile.log" resources: mem_mb=5000 script: 'scripts/build_hydro_profile.py' rule add_electricity: input: - base_network='networks/base.nc', + base_network="networks/" + RDIR + "base.nc", tech_costs=COSTS, - regions="resources/regions_onshore.geojson", - powerplants='resources/powerplants.csv', + regions="resources/" + RDIR + "regions_onshore.geojson", + powerplants="resources/" + RDIR + "powerplants.csv", hydro_capacities='data/bundle/hydro_capacities.csv', geth_hydro_capacities='data/geth2015_hydro_capacities.csv', - load='resources/load.csv', - nuts3_shapes='resources/nuts3_shapes.geojson', - **{f"profile_{tech}": f"resources/profile_{tech}.nc" + load="resources/" + RDIR + "load.csv", + nuts3_shapes="resources/" + RDIR + "nuts3_shapes.geojson", + **{f"profile_{tech}": "resources/" + RDIR + f"profile_{tech}.nc" for tech in config['renewable']}, - **{f"conventional_{carrier}_{attr}": fn for carrier, d in config.get('conventional', {None: {}}).items() for attr, fn in d.items() if str(fn).startswith("data/")}, - output: "networks/elec.nc" - log: "logs/add_electricity.log" - benchmark: "benchmarks/add_electricity" + **{f"conventional_{carrier}_{attr}": fn + for carrier, d in config.get('conventional', {None: {}}).items() + for attr, fn in d.items() if str(fn).startswith("data/")}, + output: "networks/" + RDIR + "elec.nc" + log: "logs/" + RDIR + "add_electricity.log" + benchmark: "benchmarks/" + RDIR + "add_electricity" threads: 1 resources: mem_mb=5000 script: "scripts/add_electricity.py" @@ -274,18 +281,18 @@ rule add_electricity: rule simplify_network: input: - network='networks/elec.nc', + network="networks/" + RDIR + "elec.nc", tech_costs=COSTS, - regions_onshore="resources/regions_onshore.geojson", - regions_offshore="resources/regions_offshore.geojson" + regions_onshore="resources/" + RDIR + "regions_onshore.geojson", + regions_offshore="resources/" + RDIR + "regions_offshore.geojson" output: - network='networks/elec_s{simpl}.nc', - regions_onshore="resources/regions_onshore_elec_s{simpl}.geojson", - regions_offshore="resources/regions_offshore_elec_s{simpl}.geojson", - busmap='resources/busmap_elec_s{simpl}.csv', - connection_costs='resources/connection_costs_s{simpl}.csv' - log: "logs/simplify_network/elec_s{simpl}.log" - benchmark: "benchmarks/simplify_network/elec_s{simpl}" + network="networks/" + RDIR + "elec_s{simpl}.nc", + regions_onshore="resources/" + RDIR + "regions_onshore_elec_s{simpl}.geojson", + regions_offshore="resources/" + RDIR + "regions_offshore_elec_s{simpl}.geojson", + busmap="resources/" + RDIR + "busmap_elec_s{simpl}.csv", + connection_costs="resources/" + RDIR + "connection_costs_s{simpl}.csv" + log: "logs/" + RDIR + "simplify_network/elec_s{simpl}.log" + benchmark: "benchmarks/" + RDIR + "simplify_network/elec_s{simpl}" threads: 1 resources: mem_mb=4000 script: "scripts/simplify_network.py" @@ -293,21 +300,21 @@ rule simplify_network: rule cluster_network: input: - network='networks/elec_s{simpl}.nc', - regions_onshore="resources/regions_onshore_elec_s{simpl}.geojson", - regions_offshore="resources/regions_offshore_elec_s{simpl}.geojson", - busmap=ancient('resources/busmap_elec_s{simpl}.csv'), + network="networks/" + RDIR + "elec_s{simpl}.nc", + regions_onshore="resources/" + RDIR + "regions_onshore_elec_s{simpl}.geojson", + regions_offshore="resources/" + RDIR + "regions_offshore_elec_s{simpl}.geojson", + busmap=ancient("resources/" + RDIR + "busmap_elec_s{simpl}.csv"), custom_busmap=("data/custom_busmap_elec_s{simpl}_{clusters}.csv" if config["enable"].get("custom_busmap", False) else []), tech_costs=COSTS output: - network='networks/elec_s{simpl}_{clusters}.nc', - regions_onshore="resources/regions_onshore_elec_s{simpl}_{clusters}.geojson", - regions_offshore="resources/regions_offshore_elec_s{simpl}_{clusters}.geojson", - busmap="resources/busmap_elec_s{simpl}_{clusters}.csv", - linemap="resources/linemap_elec_s{simpl}_{clusters}.csv" - log: "logs/cluster_network/elec_s{simpl}_{clusters}.log" - benchmark: "benchmarks/cluster_network/elec_s{simpl}_{clusters}" + network="networks/" + RDIR + "elec_s{simpl}_{clusters}.nc", + regions_onshore="resources/" + RDIR + "regions_onshore_elec_s{simpl}_{clusters}.geojson", + regions_offshore="resources/" + RDIR + "regions_offshore_elec_s{simpl}_{clusters}.geojson", + busmap="resources/" + RDIR + "busmap_elec_s{simpl}_{clusters}.csv", + linemap="resources/" + RDIR + "linemap_elec_s{simpl}_{clusters}.csv" + log: "logs/" + RDIR + "cluster_network/elec_s{simpl}_{clusters}.log" + benchmark: "benchmarks/" + RDIR + "cluster_network/elec_s{simpl}_{clusters}" threads: 1 resources: mem_mb=6000 script: "scripts/cluster_network.py" @@ -315,21 +322,21 @@ rule cluster_network: rule add_extra_components: input: - network='networks/elec_s{simpl}_{clusters}.nc', + network="networks/" + RDIR + "elec_s{simpl}_{clusters}.nc", tech_costs=COSTS, - output: 'networks/elec_s{simpl}_{clusters}_ec.nc' - log: "logs/add_extra_components/elec_s{simpl}_{clusters}.log" - benchmark: "benchmarks/add_extra_components/elec_s{simpl}_{clusters}_ec" + output: "networks/" + RDIR + "elec_s{simpl}_{clusters}_ec.nc" + log: "logs/" + RDIR + "add_extra_components/elec_s{simpl}_{clusters}.log" + benchmark: "benchmarks/" + RDIR + "add_extra_components/elec_s{simpl}_{clusters}_ec" threads: 1 resources: mem_mb=3000 script: "scripts/add_extra_components.py" rule prepare_network: - input: 'networks/elec_s{simpl}_{clusters}_ec.nc', tech_costs=COSTS - output: 'networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc' - log: "logs/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.log" - benchmark: "benchmarks/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" + input: "networks/" + RDIR + "elec_s{simpl}_{clusters}_ec.nc", tech_costs=COSTS, + output: "networks/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + log: "logs/" + RDIR + "prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.log" + benchmark: "benchmarks/" + RDIR + "prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 1 resources: mem_mb=4000 script: "scripts/prepare_network.py" @@ -356,13 +363,13 @@ def memory(w): rule solve_network: - input: "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + input: "networks/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + output: "results/networks/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" log: - solver=normpath("logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"), - python="logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", - memory="logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_memory.log" - benchmark: "benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" + solver=normpath("logs/" + RDIR + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"), + python="logs/" + RDIR + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", + memory="logs/" + RDIR + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_memory.log" + benchmark: "benchmarks/" + RDIR + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 4 resources: mem_mb=memory shadow: "minimal" @@ -371,14 +378,14 @@ rule solve_network: rule solve_operations_network: input: - unprepared="networks/elec_s{simpl}_{clusters}_ec.nc", - optimized="results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" + unprepared="networks/" + RDIR + "elec_s{simpl}_{clusters}_ec.nc", + optimized="results/networks/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + output: "results/networks/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" log: - solver=normpath("logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"), - python="logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", - memory="logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_memory.log" - benchmark: "benchmarks/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" + solver=normpath("logs/" + RDIR + "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"), + python="logs/" + RDIR + "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", + memory="logs/" + RDIR + "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_memory.log" + benchmark: "benchmarks/" + RDIR + "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 4 resources: mem_mb=(lambda w: 5000 + 372 * int(w.clusters)) shadow: "minimal" @@ -387,12 +394,12 @@ rule solve_operations_network: rule plot_network: input: - network="results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + network="results/networks/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", tech_costs=COSTS output: - only_map="results/plots/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}.{ext}", - ext="results/plots/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_ext.{ext}" - log: "logs/plot_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_{ext}.log" + only_map="results/plots/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}.{ext}", + ext="results/plots/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_ext.{ext}" + log: "logs/" + RDIR + "plot_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_{ext}.log" script: "scripts/plot_network.py" @@ -405,7 +412,7 @@ def input_make_summary(w): else: ll = w.ll return ([COSTS] + - expand("results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + expand("results/networks/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", ll=ll, **{k: config["scenario"][k] if getattr(w, k) == "all" else getattr(w, k) for k in ["simpl", "clusters", "opts"]})) @@ -413,30 +420,30 @@ def input_make_summary(w): rule make_summary: input: input_make_summary - output: directory("results/summaries/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}") - log: "logs/make_summary/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.log", + output: directory("results/summaries/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}") + log: "logs/" + RDIR + "make_summary/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.log", resources: mem_mb=500 script: "scripts/make_summary.py" rule plot_summary: - input: "results/summaries/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}" - output: "results/plots/summary_{summary}_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.{ext}" - log: "logs/plot_summary/{summary}_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}_{ext}.log" + input: "results/summaries/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}" + output: "results/plots/" + RDIR + "summary_{summary}_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.{ext}" + log: "logs/" + RDIR + "plot_summary/{summary}_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}_{ext}.log" resources: mem_mb=500 script: "scripts/plot_summary.py" def input_plot_p_nom_max(w): - return [("results/networks/elec_s{simpl}{maybe_cluster}.nc" + return [("results/networks/" + RDIR + "elec_s{simpl}{maybe_cluster}.nc" .format(maybe_cluster=('' if c == 'full' else ('_' + c)), **w)) for c in w.clusts.split(",")] rule plot_p_nom_max: input: input_plot_p_nom_max - output: "results/plots/elec_s{simpl}_cum_p_nom_max_{clusts}_{techs}_{country}.{ext}" - log: "logs/plot_p_nom_max/elec_s{simpl}_{clusts}_{techs}_{country}_{ext}.log" + output: "results/plots/" + RDIR + "elec_s{simpl}_cum_p_nom_max_{clusts}_{techs}_{country}.{ext}" + log: "logs/" + RDIR + "plot_p_nom_max/elec_s{simpl}_{clusts}_{techs}_{country}_{ext}.log" resources: mem_mb=500 script: "scripts/plot_p_nom_max.py" diff --git a/config.default.yaml b/config.default.yaml index 83f1cfd8..efaa14d9 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -9,6 +9,11 @@ logging: level: INFO format: '%(levelname)s:%(name)s:%(message)s' +run: + name: "" # use this to keep track of runs with different settings + dedicated_cutouts: True # set to false to use the default cutout(s) + + scenario: simpl: [''] ll: ['copt'] diff --git a/config.tutorial.yaml b/config.tutorial.yaml index ed2f9bf0..22c87834 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -9,6 +9,8 @@ logging: level: INFO format: '%(levelname)s:%(name)s:%(message)s' +run: + name: "" scenario: simpl: [''] diff --git a/doc/configuration.rst b/doc/configuration.rst index e5188da2..b59454cd 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -28,13 +28,24 @@ Top-level configuration .. _scenario: -``scenario`` -============ +``run`` +======= It is common conduct to analyse energy system optimisation models for **multiple scenarios** for a variety of reasons, e.g. assessing their sensitivity towards changing the temporal and/or geographical resolution or investigating how investment changes as more ambitious greenhouse-gas emission reduction targets are applied. +The ``run`` section is used for running and storing scenarios with different configurations which are not covered by :ref:`wildcards`. It determines the path at which resources, networks and results are stored. Therefore the user can run different configurations within the same directory. If a run with a non empty name should use the default cutouts, set ``dedicated_cutouts`` to `false`. + +.. literalinclude:: ../config.default.yaml + :language: yaml + :start-at: run: + :end-before: scenario: + + +``scenario`` +============ + The ``scenario`` section is an extraordinary section of the config file that is strongly connected to the :ref:`wildcards` and is designed to facilitate running multiple scenarios through a single command @@ -96,7 +107,7 @@ Specifies the temporal range to build an energy system model for as arguments to ``atlite`` ========== -Define and specify the ``atlite.Cutout`` used for calculating renewable potentials and time-series. All options except for ``features`` are directly used as `cutout parameters `_. +Define and specify the ``atlite.Cutout`` used for calculating renewable potentials and time-series. All options except for ``features`` are directly used as `cutout parameters `_. .. literalinclude:: ../config.default.yaml :language: yaml diff --git a/doc/release_notes.rst b/doc/release_notes.rst index dcf42355..f9cc6b44 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -14,6 +14,8 @@ Upcoming Release * When fixing line voltages to 380kV, the workflow now preserves the transmission capacity instead of the electrical impedance and reactance. +* The software now supports running the workflow with different settings within the same directory. A new config section ``run`` was created that specifies under which scenario ``name`` the created resources, networks and results should be stored. If ``name`` is not specified, the workflow uses the default paths. The entry ``dedicated_cutouts`` specifies whether the run should use cutouts from the default root directory and use run-specific cutouts. + PyPSA-Eur 0.5.0 (27th July 2022) ===================================== diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 2f25b6eb..bc01e0dc 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -8,6 +8,8 @@ logging: level: INFO format: '%(levelname)s:%(name)s:%(message)s' +run: + name: "" scenario: simpl: ['']