From 62c61438636494a9dc2a917e1706ae0df25e2d8c Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 23 Aug 2023 17:14:57 +0200 Subject: [PATCH] scenario-management: reenable shared resources, make shared resources wildcards dependent --- .github/workflows/ci.yaml | 2 +- Snakefile | 26 +- config/config.default.yaml | 5 +- config/scenarios.yaml | 26 + config/test/config.scenarios.electricity.yaml | 7 +- doc/configtables/run.csv | 8 +- doc/configtables/toplevel.csv | 1 - rules/build_electricity.smk | 184 +++---- rules/build_sector.smk | 483 ++++++++++-------- rules/collect.smk | 6 +- rules/common.smk | 4 +- rules/postprocess.smk | 2 +- rules/retrieve.smk | 4 +- rules/solve_electricity.smk | 2 +- rules/solve_myopic.smk | 16 +- rules/solve_overnight.smk | 3 +- rules/validate.smk | 20 +- scripts/__init__.py | 4 + scripts/_helpers.py | 47 +- 19 files changed, 479 insertions(+), 371 deletions(-) create mode 100644 scripts/__init__.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 68bffc4c..6cbee85c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -83,7 +83,7 @@ jobs: snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime snakemake -call all --configfile config/test/config.overnight.yaml --rerun-triggers=mtime snakemake -call all --configfile config/test/config.myopic.yaml --rerun-triggers=mtime - snakemake -call solve_elec_networks --configfile config/test/config.scenarios.electricity.yaml -n + snakemake -call solve_elec_networks --configfile config/test/config.scenarios.electricity.yaml - name: Upload artifacts uses: actions/upload-artifact@v3 diff --git a/Snakefile b/Snakefile index e495e7d3..55b90d5d 100644 --- a/Snakefile +++ b/Snakefile @@ -8,6 +8,7 @@ from pathlib import Path import yaml from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider from snakemake.utils import min_version +from scripts._helpers import path_provider min_version("7.7") HTTP = HTTPRemoteProvider() @@ -24,20 +25,23 @@ COSTS = f"data/costs_{config['costs']['year']}.csv" ATLITE_NPROCESSES = config["atlite"].get("nprocesses", 4) run = config["run"] -if run.get("scenarios", False): - if run["shared_resources"]: - raise ValueError("Cannot use shared resources with scenarios") - scenarios = yaml.safe_load(Path(config["scenariofile"]).read_text()) +scenario = run.get("scenario", {}) +if run["name"]: + if scenario.get("enable"): + fn = Path(scenario["file"]) + scenarios = yaml.safe_load(fn.read_text()) RDIR = "{run}/" -elif run["name"]: - RDIR = run["name"] + "/" else: RDIR = "" -CDIR = RDIR if not run.get("shared_cutouts") else "" +# for possibly shared resources +logs = path_provider("logs/", RDIR, run["shared_resources"]) +benchmarks = path_provider("benchmarks/", RDIR, run["shared_resources"]) +resources = path_provider("resources/", RDIR, run["shared_resources"]) + +CDIR = "" if run["shared_cutouts"] else RDIR LOGS = "logs/" + RDIR BENCHMARKS = "benchmarks/" + RDIR -RESOURCES = "resources/" + RDIR if not run.get("shared_resources") else "resources/" RESULTS = "results/" + RDIR @@ -86,9 +90,9 @@ rule dag: message: "Creating DAG of workflow." output: - dot=RESOURCES + "dag.dot", - pdf=RESOURCES + "dag.pdf", - png=RESOURCES + "dag.png", + dot=resources("dag.dot"), + pdf=resources("dag.pdf"), + png=resources("dag.png"), conda: "envs/environment.yaml" shell: diff --git a/config/config.default.yaml b/config/config.default.yaml index 5357db8d..238c3d41 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -5,7 +5,6 @@ # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#top-level-configuration version: 0.8.1 tutorial: false -scenariofile: config/scenarios.yaml logging: level: INFO @@ -22,7 +21,9 @@ remote: # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#run run: name: "" - scenarios: false + scenarios: + enable: false + file: config/scenarios.yaml disable_progressbar: false shared_resources: false shared_cutouts: true diff --git a/config/scenarios.yaml b/config/scenarios.yaml index 37d32243..c493311f 100644 --- a/config/scenarios.yaml +++ b/config/scenarios.yaml @@ -10,3 +10,29 @@ # custom-scenario: # name of the scenario # electricity: # renewable_carriers: [wind, solar] # override the list of renewable carriers + + +network2013: + snapshots: + start: "2013-01-01" + end: "2014-01-01" + inclusive: 'left' + + +network2019: + snapshots: + start: "2019-01-01" + end: "2020-01-01" + inclusive: 'left' + + renewable: + onwind: + cutout: europe-2019-era5 + offwind-ac: + cutout: europe-2019-era5 + offwind-dc: + cutout: europe-2019-era5 + solar: + cutout: europe-2019-era5 + hydro: + cutout: europe-2019-era5 diff --git a/config/test/config.scenarios.electricity.yaml b/config/test/config.scenarios.electricity.yaml index 0e4ced04..63b1892b 100644 --- a/config/test/config.scenarios.electricity.yaml +++ b/config/test/config.scenarios.electricity.yaml @@ -3,15 +3,16 @@ # SPDX-License-Identifier: CC0-1.0 tutorial: true -scenariofile: "config/test/scenarios.electricity.yaml" run: name: - test-elec-no-offshore-wind - test-elec-no-onshore-wind - scenarios: true + scenario: + enable: true + file: "config/test/scenarios.electricity.yaml" disable_progressbar: true - shared_resources: false # cannot be true if scenarios is true + shared_resources: base shared_cutouts: true scenario: diff --git a/doc/configtables/run.csv b/doc/configtables/run.csv index 2d5cf5d9..3d8e4e8c 100644 --- a/doc/configtables/run.csv +++ b/doc/configtables/run.csv @@ -1,6 +1,8 @@ ,Unit,Values,Description -name,--,str/list,"Specify a name for your run. Results will be stored under this name. If ``scenarios`` is set to ``true``, the name must contain a subset of scenario names defined in ``scenariofile``." -scenarios,--,bool,"{true, false}","Switch to select whether workflow should generate scenarios based on ``scenariofile``." +name,--,str/list,"Specify a name for your run. Results will be stored under this name. If ``scenario: enable`` is set to ``true``, the name must contain a subset of scenario names defined in ``scenario: file``." +scenario,,, +-- enable,bool,"{true, false}","Switch to select whether workflow should generate scenarios based on ``file``." +-- file,str,,Path to the scenario yaml file. The scenario file contains config overrides for each scenario. In order to be taken account, ``run:scenarios`` has to be set to ``true`` and ``run:name`` has to be a subset of top level keys given in the scenario file. In order to automatically create a `scenario.yaml` file based on a combindation of settings, alter and use the ``create_scenarios.py`` script in ``scripts``. disable_progrssbar,bool,"{true, false}","Switch to select whether progressbar should be disabled." -shared_resources,bool,"{true, false}","Switch to select whether resources should be shared across runs." +shared_resources,bool/str,,"Switch to select whether resources should be shared across runs. If a string is passed, it is assumed to be a wildcard or 'base' that indicates the cutoff after which resources are no longer shared. If 'base' is passed, resources before creating the elec.nc file are shared." shared_cutouts,bool,"{true, false}","Switch to select whether cutouts should be shared across runs." diff --git a/doc/configtables/toplevel.csv b/doc/configtables/toplevel.csv index 8cbb3e56..67954389 100644 --- a/doc/configtables/toplevel.csv +++ b/doc/configtables/toplevel.csv @@ -1,7 +1,6 @@ ,Unit,Values,Description version,--,0.x.x,Version of PyPSA-Eur. Descriptive only. tutorial,bool,"{true, false}",Switch to retrieve the tutorial data set instead of the full data set. -scenariofile,str,,Path to the scenario yaml file. The scenario file contains config overrides for each scenario. In order to be taken account, ``run:scenarios`` has to be set to ``true`` and ``run:name`` has to be a subset of top level keys given in the scenario file. In order to automatically create a `scenario.yaml` file based on a combindation of settings, alter and use the ``create_scenarios.py`` script in ``scripts``. logging,,, -- level,--,"Any of {'INFO', 'WARNING', 'ERROR'}","Restrict console outputs to all infos, warning or errors only" -- format,--,,Custom format for log messages. See `LogRecord `_ attributes. diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 5f44d2ef..be72be6d 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -8,7 +8,7 @@ if config["enable"].get("prepare_links_p_nom", False): output: "data/links_p_nom.csv", log: - LOGS + "prepare_links_p_nom.log", + logs("prepare_links_p_nom.log"), threads: 1 resources: mem_mb=1500, @@ -26,9 +26,9 @@ rule build_electricity_demand: input: ancient("data/electricity_demand.csv"), output: - RESOURCES + "load.csv", + resources("load.csv"), log: - LOGS + "build_electricity_demand.log", + logs("build_electricity_demand.log"), resources: mem_mb=5000, conda: @@ -43,12 +43,12 @@ rule build_powerplants: custom_powerplants=config_provider("electricity", "custom_powerplants"), countries=config_provider("countries"), input: - base_network=RESOURCES + "networks/base.nc", + base_network=resources("networks/base.nc"), custom_powerplants="data/custom_powerplants.csv", output: - RESOURCES + "powerplants.csv", + resources("powerplants.csv"), log: - LOGS + "build_powerplants.log", + logs("build_powerplants.log"), threads: 1 resources: mem_mb=5000, @@ -74,15 +74,15 @@ rule base_network: parameter_corrections="data/parameter_corrections.yaml", links_p_nom="data/links_p_nom.csv", links_tyndp="data/links_tyndp.csv", - country_shapes=RESOURCES + "country_shapes.geojson", - offshore_shapes=RESOURCES + "offshore_shapes.geojson", - europe_shape=RESOURCES + "europe_shape.geojson", + country_shapes=resources("country_shapes.geojson"), + offshore_shapes=resources("offshore_shapes.geojson"), + europe_shape=resources("europe_shape.geojson"), output: - RESOURCES + "networks/base.nc", + resources("networks/base.nc"), log: - LOGS + "base_network.log", + logs("base_network.log"), benchmark: - BENCHMARKS + "base_network" + benchmarks("base_network") threads: 1 resources: mem_mb=1500, @@ -104,12 +104,12 @@ rule build_shapes: ch_cantons=ancient("data/bundle/ch_cantons.csv"), ch_popgdp=ancient("data/bundle/je-e-21.03.02.xls"), output: - country_shapes=RESOURCES + "country_shapes.geojson", - offshore_shapes=RESOURCES + "offshore_shapes.geojson", - europe_shape=RESOURCES + "europe_shape.geojson", - nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", + country_shapes=resources("country_shapes.geojson"), + offshore_shapes=resources("offshore_shapes.geojson"), + europe_shape=resources("europe_shape.geojson"), + nuts3_shapes=resources("nuts3_shapes.geojson"), log: - LOGS + "build_shapes.log", + logs("build_shapes.log"), threads: 1 resources: mem_mb=1500, @@ -123,14 +123,14 @@ rule build_bus_regions: params: countries=config_provider("countries"), input: - country_shapes=RESOURCES + "country_shapes.geojson", - offshore_shapes=RESOURCES + "offshore_shapes.geojson", - base_network=RESOURCES + "networks/base.nc", + country_shapes=resources("country_shapes.geojson"), + offshore_shapes=resources("offshore_shapes.geojson"), + base_network=resources("networks/base.nc"), output: - regions_onshore=RESOURCES + "regions_onshore.geojson", - regions_offshore=RESOURCES + "regions_offshore.geojson", + regions_onshore=resources("regions_onshore.geojson"), + regions_offshore=resources("regions_offshore.geojson"), log: - LOGS + "build_bus_regions.log", + logs("build_bus_regions.log"), threads: 1 resources: mem_mb=1000, @@ -147,8 +147,8 @@ if config["enable"].get("build_cutout", False): snapshots=config_provider("snapshots"), cutouts=config_provider("atlite", "cutouts"), input: - regions_onshore=RESOURCES + "regions_onshore.geojson", - regions_offshore=RESOURCES + "regions_offshore.geojson", + regions_onshore=resources("regions_onshore.geojson"), + regions_offshore=resources("regions_offshore.geojson"), output: protected("cutouts/" + CDIR + "{cutout}.nc"), log: @@ -171,11 +171,11 @@ if config["enable"].get("build_natura_raster", False): natura=ancient("data/bundle/natura/Natura2000_end2015.shp"), cutouts=expand("cutouts/" + CDIR + "{cutouts}.nc", **config["atlite"]), output: - RESOURCES + "natura.tiff", + resources("natura.tiff"), resources: mem_mb=5000, log: - LOGS + "build_natura_raster.log", + logs("build_natura_raster.log"), conda: "../envs/environment.yaml" script: @@ -193,13 +193,13 @@ rule build_ship_raster: ], ), output: - RESOURCES + "shipdensity_raster.tif", + resources("shipdensity_raster.tif"), log: - LOGS + "build_ship_raster.log", + logs("build_ship_raster.log"), resources: mem_mb=5000, benchmark: - BENCHMARKS + "build_ship_raster" + benchmarks("build_ship_raster") conda: "../envs/environment.yaml" script: @@ -210,10 +210,10 @@ rule build_renewable_profiles: params: renewable=config_provider("renewable"), input: - base_network=RESOURCES + "networks/base.nc", + base_network=resources("networks/base.nc"), corine=ancient("data/bundle/corine/g250_clc06_V18_5.tif"), natura=lambda w: ( - RESOURCES + "natura.tiff" + resources("natura.tiff") if config_provider("renewable", w.technology, "natura")(w) else [] ), @@ -225,27 +225,27 @@ rule build_renewable_profiles: ) ), ship_density=lambda w: ( - RESOURCES + "shipdensity_raster.tif" + resources("shipdensity_raster.tif") if "ship_threshold" in config_provider("renewable", w.technology)(w).keys() else [] ), - country_shapes=RESOURCES + "country_shapes.geojson", - offshore_shapes=RESOURCES + "offshore_shapes.geojson", + country_shapes=resources("country_shapes.geojson"), + offshore_shapes=resources("offshore_shapes.geojson"), regions=lambda w: ( - RESOURCES + "regions_onshore.geojson" + resources("regions_onshore.geojson") if w.technology in ("onwind", "solar") - else RESOURCES + "regions_offshore.geojson" + else resources("regions_offshore.geojson") ), cutout=lambda w: "cutouts/" + CDIR + config_provider("renewable", w.technology, "cutout")(w) + ".nc", output: - profile=RESOURCES + "profile_{technology}.nc", + profile=resources("profile_{technology}.nc"), log: - LOGS + "build_renewable_profile_{technology}.log", + logs("build_renewable_profile_{technology}.log"), benchmark: - BENCHMARKS + "build_renewable_profiles_{technology}" + benchmarks("build_renewable_profiles_{technology}") threads: ATLITE_NPROCESSES resources: mem_mb=ATLITE_NPROCESSES * 5000, @@ -262,10 +262,10 @@ rule build_monthly_prices: co2_price_raw="data/validation/emission-spot-primary-market-auction-report-2019-data.xls", fuel_price_raw="data/validation/energy-price-trends-xlsx-5619002.xlsx", output: - co2_price=RESOURCES + "co2_price.csv", - fuel_price=RESOURCES + "monthly_fuel_price.csv", + co2_price=resources("co2_price.csv"), + fuel_price=resources("monthly_fuel_price.csv"), log: - LOGS + "build_monthly_prices.log", + logs("build_monthly_prices.log"), threads: 1 resources: mem_mb=5000, @@ -280,13 +280,13 @@ rule build_hydro_profile: hydro=config_provider("renewable", "hydro"), countries=config_provider("countries"), input: - country_shapes=RESOURCES + "country_shapes.geojson", + country_shapes=resources("country_shapes.geojson"), eia_hydro_generation="data/eia_hydro_annual_generation.csv", cutout=f"cutouts/" + CDIR + config["renewable"]["hydro"]["cutout"] + ".nc", output: - RESOURCES + "profile_hydro.nc", + resources("profile_hydro.nc"), log: - LOGS + "build_hydro_profile.log", + logs("build_hydro_profile.log"), resources: mem_mb=5000, conda: @@ -299,17 +299,17 @@ if config["lines"]["dynamic_line_rating"]["activate"]: rule build_line_rating: input: - base_network=RESOURCES + "networks/base.nc", + base_network=resources("networks/base.nc"), cutout="cutouts/" + CDIR + config["lines"]["dynamic_line_rating"]["cutout"] + ".nc", output: - output=RESOURCES + "networks/line_rating.nc", + output=resources("networks/line_rating.nc"), log: - LOGS + "build_line_rating.log", + logs("build_line_rating.log"), benchmark: - BENCHMARKS + "build_line_rating" + benchmarks("build_line_rating") threads: ATLITE_NPROCESSES resources: mem_mb=ATLITE_NPROCESSES * 1000, @@ -330,7 +330,7 @@ rule add_electricity: costs=config_provider("costs"), input: **{ - f"profile_{tech}": RESOURCES + f"profile_{tech}.nc" + f"profile_{tech}": resources(f"profile_{tech}.nc") for tech in config["electricity"]["renewable_carriers"] }, **{ @@ -340,27 +340,27 @@ rule add_electricity: for attr, fn in d.items() if str(fn).startswith("data/") }, - base_network=RESOURCES + "networks/base.nc", - line_rating=RESOURCES + "networks/line_rating.nc" + base_network=resources("networks/base.nc"), + line_rating=resources("networks/line_rating.nc") if config["lines"]["dynamic_line_rating"]["activate"] - else RESOURCES + "networks/base.nc", + else resources("networks/base.nc"), tech_costs=COSTS, - regions=RESOURCES + "regions_onshore.geojson", - powerplants=RESOURCES + "powerplants.csv", + regions=resources("regions_onshore.geojson"), + powerplants=resources("powerplants.csv"), hydro_capacities=ancient("data/bundle/hydro_capacities.csv"), geth_hydro_capacities="data/geth2015_hydro_capacities.csv", unit_commitment="data/unit_commitment.csv", - fuel_price=RESOURCES + "monthly_fuel_price.csv" + fuel_price=resources("monthly_fuel_price.csv") if config["conventional"]["dynamic_fuel_price"] else [], - load=RESOURCES + "load.csv", - nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", + load=resources("load.csv"), + nuts3_shapes=resources("nuts3_shapes.geojson"), output: - RESOURCES + "networks/elec.nc", + resources("networks/elec.nc"), log: - LOGS + "add_electricity.log", + logs("add_electricity.log"), benchmark: - BENCHMARKS + "add_electricity" + benchmarks("add_electricity") threads: 1 resources: mem_mb=10000, @@ -383,20 +383,20 @@ rule simplify_network: p_max_pu=config_provider("links", "p_max_pu", default=1.0), costs=config_provider("costs"), input: - network=RESOURCES + "networks/elec.nc", + network=resources("networks/elec.nc"), tech_costs=COSTS, - regions_onshore=RESOURCES + "regions_onshore.geojson", - regions_offshore=RESOURCES + "regions_offshore.geojson", + regions_onshore=resources("regions_onshore.geojson"), + regions_offshore=resources("regions_offshore.geojson"), output: - network=RESOURCES + "networks/elec_s{simpl}.nc", - regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson", - regions_offshore=RESOURCES + "regions_offshore_elec_s{simpl}.geojson", - busmap=RESOURCES + "busmap_elec_s{simpl}.csv", - connection_costs=RESOURCES + "connection_costs_s{simpl}.csv", + network=resources("networks/elec_s{simpl}.nc"), + regions_onshore=resources("regions_onshore_elec_s{simpl}.geojson"), + regions_offshore=resources("regions_offshore_elec_s{simpl}.geojson"), + busmap=resources("busmap_elec_s{simpl}.csv"), + connection_costs=resources("connection_costs_s{simpl}.csv"), log: - LOGS + "simplify_network/elec_s{simpl}.log", + logs("simplify_network/elec_s{simpl}.log"), benchmark: - BENCHMARKS + "simplify_network/elec_s{simpl}" + benchmarks("simplify_network/elec_s{simpl}") threads: 1 resources: mem_mb=12000, @@ -422,10 +422,10 @@ rule cluster_network: length_factor=config_provider("lines", "length_factor"), costs=config_provider("costs"), input: - network=RESOURCES + "networks/elec_s{simpl}.nc", - regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson", - regions_offshore=RESOURCES + "regions_offshore_elec_s{simpl}.geojson", - busmap=ancient(RESOURCES + "busmap_elec_s{simpl}.csv"), + network=resources("networks/elec_s{simpl}.nc"), + regions_onshore=resources("regions_onshore_elec_s{simpl}.geojson"), + regions_offshore=resources("regions_offshore_elec_s{simpl}.geojson"), + busmap=ancient(resources("busmap_elec_s{simpl}.csv")), custom_busmap=( "data/custom_busmap_elec_s{simpl}_{clusters}.csv" if config["enable"].get("custom_busmap", False) @@ -433,15 +433,15 @@ rule cluster_network: ), tech_costs=COSTS, output: - network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", - regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", - regions_offshore=RESOURCES + "regions_offshore_elec_s{simpl}_{clusters}.geojson", - busmap=RESOURCES + "busmap_elec_s{simpl}_{clusters}.csv", - linemap=RESOURCES + "linemap_elec_s{simpl}_{clusters}.csv", + network=resources("networks/elec_s{simpl}_{clusters}.nc"), + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), + regions_offshore=resources("regions_offshore_elec_s{simpl}_{clusters}.geojson"), + busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"), + linemap=resources("linemap_elec_s{simpl}_{clusters}.csv"), log: - LOGS + "cluster_network/elec_s{simpl}_{clusters}.log", + logs("cluster_network/elec_s{simpl}_{clusters}.log"), benchmark: - BENCHMARKS + "cluster_network/elec_s{simpl}_{clusters}" + benchmarks("cluster_network/elec_s{simpl}_{clusters}") threads: 1 resources: mem_mb=10000, @@ -457,14 +457,14 @@ rule add_extra_components: max_hours=config_provider("electricity", "max_hours"), costs=config_provider("costs"), input: - network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", + network=resources("networks/elec_s{simpl}_{clusters}.nc"), tech_costs=COSTS, output: - RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", + resources("networks/elec_s{simpl}_{clusters}_ec.nc"), log: - LOGS + "add_extra_components/elec_s{simpl}_{clusters}.log", + logs("add_extra_components/elec_s{simpl}_{clusters}.log"), benchmark: - BENCHMARKS + "add_extra_components/elec_s{simpl}_{clusters}_ec" + benchmarks("add_extra_components/elec_s{simpl}_{clusters}_ec") threads: 1 resources: mem_mb=4000, @@ -484,15 +484,15 @@ rule prepare_network: max_hours=config_provider("electricity", "max_hours"), costs=config_provider("costs"), input: - RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", + resources("networks/elec_s{simpl}_{clusters}_ec.nc"), tech_costs=COSTS, - co2_price=RESOURCES + "co2_price.csv", + co2_price=resources("co2_price.csv"), output: - RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"), log: - LOGS + "prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.log", + logs("prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.log"), benchmark: - (BENCHMARKS + "prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}") + (benchmarks("prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}")) threads: 1 resources: mem_mb=4000, diff --git a/rules/build_sector.smk b/rules/build_sector.smk index cc80ab64..483dcdb7 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -5,19 +5,19 @@ rule build_population_layouts: input: - nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", + nuts3_shapes=resources("nuts3_shapes.geojson"), urban_percent="data/urban_percent.csv", cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", output: - pop_layout_total=RESOURCES + "pop_layout_total.nc", - pop_layout_urban=RESOURCES + "pop_layout_urban.nc", - pop_layout_rural=RESOURCES + "pop_layout_rural.nc", + pop_layout_total=resources("pop_layout_total.nc"), + pop_layout_urban=resources("pop_layout_urban.nc"), + pop_layout_rural=resources("pop_layout_rural.nc"), log: - LOGS + "build_population_layouts.log", + logs("build_population_layouts.log"), resources: mem_mb=20000, benchmark: - BENCHMARKS + "build_population_layouts" + benchmarks("build_population_layouts") threads: 8 conda: "../envs/environment.yaml" @@ -27,19 +27,19 @@ rule build_population_layouts: rule build_clustered_population_layouts: input: - pop_layout_total=RESOURCES + "pop_layout_total.nc", - pop_layout_urban=RESOURCES + "pop_layout_urban.nc", - pop_layout_rural=RESOURCES + "pop_layout_rural.nc", - regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", + pop_layout_total=resources("pop_layout_total.nc"), + pop_layout_urban=resources("pop_layout_urban.nc"), + pop_layout_rural=resources("pop_layout_rural.nc"), + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", output: - clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", + clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), log: - LOGS + "build_clustered_population_layouts_{simpl}_{clusters}.log", + logs("build_clustered_population_layouts_{simpl}_{clusters}.log"), resources: mem_mb=10000, benchmark: - BENCHMARKS + "build_clustered_population_layouts/s{simpl}_{clusters}" + benchmarks("build_clustered_population_layouts/s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -48,19 +48,19 @@ rule build_clustered_population_layouts: rule build_simplified_population_layouts: input: - pop_layout_total=RESOURCES + "pop_layout_total.nc", - pop_layout_urban=RESOURCES + "pop_layout_urban.nc", - pop_layout_rural=RESOURCES + "pop_layout_rural.nc", - regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson", + pop_layout_total=resources("pop_layout_total.nc"), + pop_layout_urban=resources("pop_layout_urban.nc"), + pop_layout_rural=resources("pop_layout_rural.nc"), + regions_onshore=resources("regions_onshore_elec_s{simpl}.geojson"), cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", output: - clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}.csv", + clustered_pop_layout=resources("pop_layout_elec_s{simpl}.csv"), resources: mem_mb=10000, log: - LOGS + "build_simplified_population_layouts_{simpl}", + logs("build_simplified_population_layouts_{simpl}"), benchmark: - BENCHMARKS + "build_simplified_population_layouts/s{simpl}" + benchmarks("build_simplified_population_layouts/s{simpl}") conda: "../envs/environment.yaml" script: @@ -73,11 +73,11 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: input: gas_network="data/gas_network/scigrid-gas/data/IGGIELGN_PipeSegments.geojson", output: - cleaned_gas_network=RESOURCES + "gas_network.csv", + cleaned_gas_network=resources("gas_network.csv"), resources: mem_mb=4000, log: - LOGS + "build_gas_network.log", + logs("build_gas_network.log"), conda: "../envs/environment.yaml" script: @@ -91,19 +91,21 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: ), entry="data/gas_network/scigrid-gas/data/IGGIELGN_BorderPoints.geojson", production="data/gas_network/scigrid-gas/data/IGGIELGN_Productions.geojson", - regions_onshore=RESOURCES - + "regions_onshore_elec_s{simpl}_{clusters}.geojson", - regions_offshore=RESOURCES - + "regions_offshore_elec_s{simpl}_{clusters}.geojson", + regions_onshore=resources( + "regions_onshore_elec_s{simpl}_{clusters}.geojson" + ), + regions_offshore=resources( + "regions_offshore_elec_s{simpl}_{clusters}.geojson" + ), output: - gas_input_nodes=RESOURCES - + "gas_input_locations_s{simpl}_{clusters}.geojson", - gas_input_nodes_simplified=RESOURCES - + "gas_input_locations_s{simpl}_{clusters}_simplified.csv", + gas_input_nodes=resources("gas_input_locations_s{simpl}_{clusters}.geojson"), + gas_input_nodes_simplified=resources( + "gas_input_locations_s{simpl}_{clusters}_simplified.csv" + ), resources: mem_mb=2000, log: - LOGS + "build_gas_input_locations_s{simpl}_{clusters}.log", + logs("build_gas_input_locations_s{simpl}_{clusters}.log"), conda: "../envs/environment.yaml" script: @@ -111,17 +113,19 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: rule cluster_gas_network: input: - cleaned_gas_network=RESOURCES + "gas_network.csv", - regions_onshore=RESOURCES - + "regions_onshore_elec_s{simpl}_{clusters}.geojson", - regions_offshore=RESOURCES - + "regions_offshore_elec_s{simpl}_{clusters}.geojson", + cleaned_gas_network=resources("gas_network.csv"), + regions_onshore=resources( + "regions_onshore_elec_s{simpl}_{clusters}.geojson" + ), + regions_offshore=resources( + "regions_offshore_elec_s{simpl}_{clusters}.geojson" + ), output: - clustered_gas_network=RESOURCES + "gas_network_elec_s{simpl}_{clusters}.csv", + clustered_gas_network=resources("gas_network_elec_s{simpl}_{clusters}.csv"), resources: mem_mb=4000, log: - LOGS + "cluster_gas_network_s{simpl}_{clusters}.log", + logs("cluster_gas_network_s{simpl}_{clusters}.log"), conda: "../envs/environment.yaml" script: @@ -143,18 +147,18 @@ rule build_heat_demands: params: snapshots=config_provider("snapshots"), input: - pop_layout=RESOURCES + "pop_layout_{scope}.nc", - regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", + pop_layout=resources("pop_layout_{scope}.nc"), + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", output: - heat_demand=RESOURCES + "heat_demand_{scope}_elec_s{simpl}_{clusters}.nc", + heat_demand=resources("heat_demand_{scope}_elec_s{simpl}_{clusters}.nc"), resources: mem_mb=20000, threads: 8 log: - LOGS + "build_heat_demands_{scope}_{simpl}_{clusters}.loc", + logs("build_heat_demands_{scope}_{simpl}_{clusters}.loc"), benchmark: - BENCHMARKS + "build_heat_demands/{scope}_s{simpl}_{clusters}" + benchmarks("build_heat_demands/{scope}_s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -165,19 +169,19 @@ rule build_temperature_profiles: params: snapshots=config_provider("snapshots"), input: - pop_layout=RESOURCES + "pop_layout_{scope}.nc", - regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", + pop_layout=resources("pop_layout_{scope}.nc"), + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", output: - temp_soil=RESOURCES + "temp_soil_{scope}_elec_s{simpl}_{clusters}.nc", - temp_air=RESOURCES + "temp_air_{scope}_elec_s{simpl}_{clusters}.nc", + temp_soil=resources("temp_soil_{scope}_elec_s{simpl}_{clusters}.nc"), + temp_air=resources("temp_air_{scope}_elec_s{simpl}_{clusters}.nc"), resources: mem_mb=20000, threads: 8 log: - LOGS + "build_temperature_profiles_{scope}_{simpl}_{clusters}.log", + logs("build_temperature_profiles_{scope}_{simpl}_{clusters}.log"), benchmark: - BENCHMARKS + "build_temperature_profiles/{scope}_s{simpl}_{clusters}" + benchmarks("build_temperature_profiles/{scope}_s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -188,25 +192,25 @@ rule build_cop_profiles: params: heat_pump_sink_T=config_provider("sector", "heat_pump_sink_T"), input: - temp_soil_total=RESOURCES + "temp_soil_total_elec_s{simpl}_{clusters}.nc", - temp_soil_rural=RESOURCES + "temp_soil_rural_elec_s{simpl}_{clusters}.nc", - temp_soil_urban=RESOURCES + "temp_soil_urban_elec_s{simpl}_{clusters}.nc", - temp_air_total=RESOURCES + "temp_air_total_elec_s{simpl}_{clusters}.nc", - temp_air_rural=RESOURCES + "temp_air_rural_elec_s{simpl}_{clusters}.nc", - temp_air_urban=RESOURCES + "temp_air_urban_elec_s{simpl}_{clusters}.nc", + temp_soil_total=resources("temp_soil_total_elec_s{simpl}_{clusters}.nc"), + temp_soil_rural=resources("temp_soil_rural_elec_s{simpl}_{clusters}.nc"), + temp_soil_urban=resources("temp_soil_urban_elec_s{simpl}_{clusters}.nc"), + temp_air_total=resources("temp_air_total_elec_s{simpl}_{clusters}.nc"), + temp_air_rural=resources("temp_air_rural_elec_s{simpl}_{clusters}.nc"), + temp_air_urban=resources("temp_air_urban_elec_s{simpl}_{clusters}.nc"), output: - cop_soil_total=RESOURCES + "cop_soil_total_elec_s{simpl}_{clusters}.nc", - cop_soil_rural=RESOURCES + "cop_soil_rural_elec_s{simpl}_{clusters}.nc", - cop_soil_urban=RESOURCES + "cop_soil_urban_elec_s{simpl}_{clusters}.nc", - cop_air_total=RESOURCES + "cop_air_total_elec_s{simpl}_{clusters}.nc", - cop_air_rural=RESOURCES + "cop_air_rural_elec_s{simpl}_{clusters}.nc", - cop_air_urban=RESOURCES + "cop_air_urban_elec_s{simpl}_{clusters}.nc", + cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"), + cop_soil_rural=resources("cop_soil_rural_elec_s{simpl}_{clusters}.nc"), + cop_soil_urban=resources("cop_soil_urban_elec_s{simpl}_{clusters}.nc"), + cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"), + cop_air_rural=resources("cop_air_rural_elec_s{simpl}_{clusters}.nc"), + cop_air_urban=resources("cop_air_urban_elec_s{simpl}_{clusters}.nc"), resources: mem_mb=20000, log: - LOGS + "build_cop_profiles_s{simpl}_{clusters}.log", + logs("build_cop_profiles_s{simpl}_{clusters}.log"), benchmark: - BENCHMARKS + "build_cop_profiles/s{simpl}_{clusters}" + benchmarks("build_cop_profiles/s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -218,18 +222,18 @@ rule build_solar_thermal_profiles: snapshots=config_provider("snapshots"), solar_thermal=config_provider("solar_thermal"), input: - pop_layout=RESOURCES + "pop_layout_{scope}.nc", - regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", + pop_layout=resources("pop_layout_{scope}.nc"), + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), cutout="cutouts/" + CDIR + config["atlite"]["default_cutout"] + ".nc", output: - solar_thermal=RESOURCES + "solar_thermal_{scope}_elec_s{simpl}_{clusters}.nc", + solar_thermal=resources("solar_thermal_{scope}_elec_s{simpl}_{clusters}.nc"), resources: mem_mb=20000, threads: 16 log: - LOGS + "build_solar_thermal_profiles_{scope}_s{simpl}_{clusters}.log", + logs("build_solar_thermal_profiles_{scope}_s{simpl}_{clusters}.log"), benchmark: - BENCHMARKS + "build_solar_thermal_profiles/{scope}_s{simpl}_{clusters}" + benchmarks("build_solar_thermal_profiles/{scope}_s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -241,23 +245,23 @@ rule build_energy_totals: countries=config_provider("countries"), energy=config_provider("energy"), input: - nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", + nuts3_shapes=resources("nuts3_shapes.geojson"), co2="data/eea/UNFCCC_v23.csv", swiss="data/switzerland-sfoe/switzerland-new_format.csv", idees="data/jrc-idees-2015", district_heat_share="data/district_heat_share.csv", eurostat=input_eurostat, output: - energy_name=RESOURCES + "energy_totals.csv", - co2_name=RESOURCES + "co2_totals.csv", - transport_name=RESOURCES + "transport_data.csv", + energy_name=resources("energy_totals.csv"), + co2_name=resources("co2_totals.csv"), + transport_name=resources("transport_data.csv"), threads: 16 resources: mem_mb=10000, log: - LOGS + "build_energy_totals.log", + logs("build_energy_totals.log"), benchmark: - BENCHMARKS + "build_energy_totals" + benchmarks("build_energy_totals") conda: "../envs/environment.yaml" script: @@ -273,22 +277,23 @@ rule build_biomass_potentials: keep_local=True, ), nuts2="data/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21 - regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), nuts3_population=ancient("data/bundle/nama_10r_3popgdp.tsv.gz"), swiss_cantons=ancient("data/bundle/ch_cantons.csv"), swiss_population=ancient("data/bundle/je-e-21.03.02.xls"), - country_shapes=RESOURCES + "country_shapes.geojson", + country_shapes=resources("country_shapes.geojson"), output: - biomass_potentials_all=RESOURCES - + "biomass_potentials_all_s{simpl}_{clusters}.csv", - biomass_potentials=RESOURCES + "biomass_potentials_s{simpl}_{clusters}.csv", + biomass_potentials_all=resources( + "biomass_potentials_all_s{simpl}_{clusters}.csv" + ), + biomass_potentials=resources("biomass_potentials_s{simpl}_{clusters}.csv"), threads: 1 resources: mem_mb=1000, log: - LOGS + "build_biomass_potentials_s{simpl}_{clusters}.log", + logs("build_biomass_potentials_s{simpl}_{clusters}.log"), benchmark: - BENCHMARKS + "build_biomass_potentials_s{simpl}_{clusters}" + benchmarks("build_biomass_potentials_s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -304,14 +309,14 @@ if config["sector"]["biomass_transport"] or config["sector"]["biomass_spatial"]: keep_local=True, ), output: - biomass_transport_costs=RESOURCES + "biomass_transport_costs.csv", + biomass_transport_costs=resources("biomass_transport_costs.csv"), threads: 1 resources: mem_mb=1000, log: - LOGS + "build_biomass_transport_costs.log", + logs("build_biomass_transport_costs.log"), benchmark: - BENCHMARKS + "build_biomass_transport_costs" + benchmarks("build_biomass_transport_costs") conda: "../envs/environment.yaml" script: @@ -337,20 +342,23 @@ if config["sector"]["regional_co2_sequestration_potential"]["enable"]: "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson", keep_local=True, ), - regions_onshore=RESOURCES - + "regions_onshore_elec_s{simpl}_{clusters}.geojson", - regions_offshore=RESOURCES - + "regions_offshore_elec_s{simpl}_{clusters}.geojson", + regions_onshore=resources( + "regions_onshore_elec_s{simpl}_{clusters}.geojson" + ), + regions_offshore=resources( + "regions_offshore_elec_s{simpl}_{clusters}.geojson" + ), output: - sequestration_potential=RESOURCES - + "co2_sequestration_potential_elec_s{simpl}_{clusters}.csv", + sequestration_potential=resources( + "co2_sequestration_potential_elec_s{simpl}_{clusters}.csv" + ), threads: 1 resources: mem_mb=4000, log: - LOGS + "build_sequestration_potentials_s{simpl}_{clusters}.log", + logs("build_sequestration_potentials_s{simpl}_{clusters}.log"), benchmark: - BENCHMARKS + "build_sequestration_potentials_s{simpl}_{clusters}" + benchmarks("build_sequestration_potentials_s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -367,17 +375,17 @@ if not config["sector"]["regional_co2_sequestration_potential"]["enable"]: rule build_salt_cavern_potentials: input: salt_caverns="data/h2_salt_caverns_GWh_per_sqkm.geojson", - regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", - regions_offshore=RESOURCES + "regions_offshore_elec_s{simpl}_{clusters}.geojson", + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), + regions_offshore=resources("regions_offshore_elec_s{simpl}_{clusters}.geojson"), output: - h2_cavern_potential=RESOURCES + "salt_cavern_potentials_s{simpl}_{clusters}.csv", + h2_cavern_potential=resources("salt_cavern_potentials_s{simpl}_{clusters}.csv"), threads: 1 resources: mem_mb=2000, log: - LOGS + "build_salt_cavern_potentials_s{simpl}_{clusters}.log", + logs("build_salt_cavern_potentials_s{simpl}_{clusters}.log"), benchmark: - BENCHMARKS + "build_salt_cavern_potentials_s{simpl}_{clusters}" + benchmarks("build_salt_cavern_potentials_s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -390,14 +398,14 @@ rule build_ammonia_production: input: usgs="data/myb1-2017-nitro.xls", output: - ammonia_production=RESOURCES + "ammonia_production.csv", + ammonia_production=resources("ammonia_production.csv"), threads: 1 resources: mem_mb=1000, log: - LOGS + "build_ammonia_production.log", + logs("build_ammonia_production.log"), benchmark: - BENCHMARKS + "build_ammonia_production" + benchmarks("build_ammonia_production") conda: "../envs/environment.yaml" script: @@ -409,17 +417,17 @@ rule build_industry_sector_ratios: industry=config_provider("industry"), ammonia=config_provider("sector", "ammonia", default=False), input: - ammonia_production=RESOURCES + "ammonia_production.csv", + ammonia_production=resources("ammonia_production.csv"), idees="data/jrc-idees-2015", output: - industry_sector_ratios=RESOURCES + "industry_sector_ratios.csv", + industry_sector_ratios=resources("industry_sector_ratios.csv"), threads: 1 resources: mem_mb=1000, log: - LOGS + "build_industry_sector_ratios.log", + logs("build_industry_sector_ratios.log"), benchmark: - BENCHMARKS + "build_industry_sector_ratios" + benchmarks("build_industry_sector_ratios") conda: "../envs/environment.yaml" script: @@ -431,19 +439,20 @@ rule build_industrial_production_per_country: industry=config_provider("industry"), countries=config_provider("countries"), input: - ammonia_production=RESOURCES + "ammonia_production.csv", + ammonia_production=resources("ammonia_production.csv"), jrc="data/jrc-idees-2015", eurostat="data/eurostat-energy_balances-may_2018_edition", output: - industrial_production_per_country=RESOURCES - + "industrial_production_per_country.csv", + industrial_production_per_country=resources( + "industrial_production_per_country.csv" + ), threads: 8 resources: mem_mb=1000, log: - LOGS + "build_industrial_production_per_country.log", + logs("build_industrial_production_per_country.log"), benchmark: - BENCHMARKS + "build_industrial_production_per_country" + benchmarks("build_industrial_production_per_country") conda: "../envs/environment.yaml" script: @@ -454,21 +463,23 @@ rule build_industrial_production_per_country_tomorrow: params: industry=config_provider("industry"), input: - industrial_production_per_country=RESOURCES - + "industrial_production_per_country.csv", + industrial_production_per_country=resources( + "industrial_production_per_country.csv" + ), output: - industrial_production_per_country_tomorrow=RESOURCES - + "industrial_production_per_country_tomorrow_{planning_horizons}.csv", + industrial_production_per_country_tomorrow=resources( + "industrial_production_per_country_tomorrow_{planning_horizons}.csv" + ), threads: 1 resources: mem_mb=1000, log: - LOGS - + "build_industrial_production_per_country_tomorrow_{planning_horizons}.log", + logs("build_industrial_production_per_country_tomorrow_{planning_horizons}.log"), benchmark: ( - BENCHMARKS - + "build_industrial_production_per_country_tomorrow_{planning_horizons}" + benchmarks( + "build_industrial_production_per_country_tomorrow_{planning_horizons}" + ) ) conda: "../envs/environment.yaml" @@ -483,19 +494,20 @@ rule build_industrial_distribution_key: ), countries=config_provider("countries"), input: - regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", - clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", + regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), + clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), hotmaps_industrial_database="data/Industrial_Database.csv", output: - industrial_distribution_key=RESOURCES - + "industrial_distribution_key_elec_s{simpl}_{clusters}.csv", + industrial_distribution_key=resources( + "industrial_distribution_key_elec_s{simpl}_{clusters}.csv" + ), threads: 1 resources: mem_mb=1000, log: - LOGS + "build_industrial_distribution_key_s{simpl}_{clusters}.log", + logs("build_industrial_distribution_key_s{simpl}_{clusters}.log"), benchmark: - BENCHMARKS + "build_industrial_distribution_key/s{simpl}_{clusters}" + benchmarks("build_industrial_distribution_key/s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -504,23 +516,28 @@ rule build_industrial_distribution_key: rule build_industrial_production_per_node: input: - industrial_distribution_key=RESOURCES - + "industrial_distribution_key_elec_s{simpl}_{clusters}.csv", - industrial_production_per_country_tomorrow=RESOURCES - + "industrial_production_per_country_tomorrow_{planning_horizons}.csv", + industrial_distribution_key=resources( + "industrial_distribution_key_elec_s{simpl}_{clusters}.csv" + ), + industrial_production_per_country_tomorrow=resources( + "industrial_production_per_country_tomorrow_{planning_horizons}.csv" + ), output: - industrial_production_per_node=RESOURCES - + "industrial_production_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + industrial_production_per_node=resources( + "industrial_production_elec_s{simpl}_{clusters}_{planning_horizons}.csv" + ), threads: 1 resources: mem_mb=1000, log: - LOGS - + "build_industrial_production_per_node_s{simpl}_{clusters}_{planning_horizons}.log", + logs( + "build_industrial_production_per_node_s{simpl}_{clusters}_{planning_horizons}.log" + ), benchmark: ( - BENCHMARKS - + "build_industrial_production_per_node/s{simpl}_{clusters}_{planning_horizons}" + benchmarks( + "build_industrial_production_per_node/s{simpl}_{clusters}_{planning_horizons}" + ) ) conda: "../envs/environment.yaml" @@ -530,24 +547,29 @@ rule build_industrial_production_per_node: rule build_industrial_energy_demand_per_node: input: - industry_sector_ratios=RESOURCES + "industry_sector_ratios.csv", - industrial_production_per_node=RESOURCES - + "industrial_production_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - industrial_energy_demand_per_node_today=RESOURCES - + "industrial_energy_demand_today_elec_s{simpl}_{clusters}.csv", + industry_sector_ratios=resources("industry_sector_ratios.csv"), + industrial_production_per_node=resources( + "industrial_production_elec_s{simpl}_{clusters}_{planning_horizons}.csv" + ), + industrial_energy_demand_per_node_today=resources( + "industrial_energy_demand_today_elec_s{simpl}_{clusters}.csv" + ), output: - industrial_energy_demand_per_node=RESOURCES - + "industrial_energy_demand_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + industrial_energy_demand_per_node=resources( + "industrial_energy_demand_elec_s{simpl}_{clusters}_{planning_horizons}.csv" + ), threads: 1 resources: mem_mb=1000, log: - LOGS - + "build_industrial_energy_demand_per_node_s{simpl}_{clusters}_{planning_horizons}.log", + logs( + "build_industrial_energy_demand_per_node_s{simpl}_{clusters}_{planning_horizons}.log" + ), benchmark: ( - BENCHMARKS - + "build_industrial_energy_demand_per_node/s{simpl}_{clusters}_{planning_horizons}" + benchmarks( + "build_industrial_energy_demand_per_node/s{simpl}_{clusters}_{planning_horizons}" + ) ) conda: "../envs/environment.yaml" @@ -561,19 +583,21 @@ rule build_industrial_energy_demand_per_country_today: industry=config_provider("industry"), input: jrc="data/jrc-idees-2015", - ammonia_production=RESOURCES + "ammonia_production.csv", - industrial_production_per_country=RESOURCES - + "industrial_production_per_country.csv", + ammonia_production=resources("ammonia_production.csv"), + industrial_production_per_country=resources( + "industrial_production_per_country.csv" + ), output: - industrial_energy_demand_per_country_today=RESOURCES - + "industrial_energy_demand_per_country_today.csv", + industrial_energy_demand_per_country_today=resources( + "industrial_energy_demand_per_country_today.csv" + ), threads: 8 resources: mem_mb=1000, log: - LOGS + "build_industrial_energy_demand_per_country_today.log", + logs("build_industrial_energy_demand_per_country_today.log"), benchmark: - BENCHMARKS + "build_industrial_energy_demand_per_country_today" + benchmarks("build_industrial_energy_demand_per_country_today") conda: "../envs/environment.yaml" script: @@ -582,20 +606,23 @@ rule build_industrial_energy_demand_per_country_today: rule build_industrial_energy_demand_per_node_today: input: - industrial_distribution_key=RESOURCES - + "industrial_distribution_key_elec_s{simpl}_{clusters}.csv", - industrial_energy_demand_per_country_today=RESOURCES - + "industrial_energy_demand_per_country_today.csv", + industrial_distribution_key=resources( + "industrial_distribution_key_elec_s{simpl}_{clusters}.csv" + ), + industrial_energy_demand_per_country_today=resources( + "industrial_energy_demand_per_country_today.csv" + ), output: - industrial_energy_demand_per_node_today=RESOURCES - + "industrial_energy_demand_today_elec_s{simpl}_{clusters}.csv", + industrial_energy_demand_per_node_today=resources( + "industrial_energy_demand_today_elec_s{simpl}_{clusters}.csv" + ), threads: 1 resources: mem_mb=1000, log: - LOGS + "build_industrial_energy_demand_per_node_today_s{simpl}_{clusters}.log", + logs("build_industrial_energy_demand_per_node_today_s{simpl}_{clusters}.log"), benchmark: - BENCHMARKS + "build_industrial_energy_demand_per_node_today/s{simpl}_{clusters}" + benchmarks("build_industrial_energy_demand_per_node_today/s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -611,23 +638,23 @@ if config["sector"]["retrofitting"]["retro_endogen"]: input: building_stock="data/retro/data_building_stock.csv", data_tabula="data/retro/tabula-calculator-calcsetbuilding.csv", - air_temperature=RESOURCES + "temp_air_total_elec_s{simpl}_{clusters}.nc", + air_temperature=resources("temp_air_total_elec_s{simpl}_{clusters}.nc"), u_values_PL="data/retro/u_values_poland.csv", tax_w="data/retro/electricity_taxes_eu.csv", construction_index="data/retro/comparative_level_investment.csv", floor_area_missing="data/retro/floor_area_missing.csv", - clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", + clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), cost_germany="data/retro/retro_cost_germany.csv", window_assumptions="data/retro/window_assumptions.csv", output: - retro_cost=RESOURCES + "retro_cost_elec_s{simpl}_{clusters}.csv", - floor_area=RESOURCES + "floor_area_elec_s{simpl}_{clusters}.csv", + retro_cost=resources("retro_cost_elec_s{simpl}_{clusters}.csv"), + floor_area=resources("floor_area_elec_s{simpl}_{clusters}.csv"), resources: mem_mb=1000, log: - LOGS + "build_retro_cost_s{simpl}_{clusters}.log", + logs("build_retro_cost_s{simpl}_{clusters}.log"), benchmark: - BENCHMARKS + "build_retro_cost/s{simpl}_{clusters}" + benchmarks("build_retro_cost/s{simpl}_{clusters}") conda: "../envs/environment.yaml" script: @@ -643,15 +670,15 @@ if not config["sector"]["retrofitting"]["retro_endogen"]: rule build_population_weighted_energy_totals: input: - energy_totals=RESOURCES + "energy_totals.csv", - clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", + energy_totals=resources("energy_totals.csv"), + clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), output: - RESOURCES + "pop_weighted_energy_totals_s{simpl}_{clusters}.csv", + resources("pop_weighted_energy_totals_s{simpl}_{clusters}.csv"), threads: 1 resources: mem_mb=2000, log: - LOGS + "build_population_weighted_energy_totals_s{simpl}_{clusters}.log", + logs("build_population_weighted_energy_totals_s{simpl}_{clusters}.log"), conda: "../envs/environment.yaml" script: @@ -661,16 +688,16 @@ rule build_population_weighted_energy_totals: rule build_shipping_demand: input: ports="data/attributed_ports.json", - scope=RESOURCES + "europe_shape.geojson", - regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", - demand=RESOURCES + "energy_totals.csv", + scope=resources("europe_shape.geojson"), + regions=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), + demand=resources("energy_totals.csv"), output: - RESOURCES + "shipping_demand_s{simpl}_{clusters}.csv", + resources("shipping_demand_s{simpl}_{clusters}.csv"), threads: 1 resources: mem_mb=2000, log: - LOGS + "build_shipping_demand_s{simpl}_{clusters}.log", + logs("build_shipping_demand_s{simpl}_{clusters}.log"), conda: "../envs/environment.yaml" script: @@ -682,23 +709,24 @@ rule build_transport_demand: snapshots=config_provider("snapshots"), sector=config_provider("sector"), input: - clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", - pop_weighted_energy_totals=RESOURCES - + "pop_weighted_energy_totals_s{simpl}_{clusters}.csv", - transport_data=RESOURCES + "transport_data.csv", + clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), + pop_weighted_energy_totals=resources( + "pop_weighted_energy_totals_s{simpl}_{clusters}.csv" + ), + transport_data=resources("transport_data.csv"), traffic_data_KFZ="data/emobility/KFZ__count", traffic_data_Pkw="data/emobility/Pkw__count", - temp_air_total=RESOURCES + "temp_air_total_elec_s{simpl}_{clusters}.nc", + temp_air_total=resources("temp_air_total_elec_s{simpl}_{clusters}.nc"), output: - transport_demand=RESOURCES + "transport_demand_s{simpl}_{clusters}.csv", - transport_data=RESOURCES + "transport_data_s{simpl}_{clusters}.csv", - avail_profile=RESOURCES + "avail_profile_s{simpl}_{clusters}.csv", - dsm_profile=RESOURCES + "dsm_profile_s{simpl}_{clusters}.csv", + transport_demand=resources("transport_demand_s{simpl}_{clusters}.csv"), + transport_data=resources("transport_data_s{simpl}_{clusters}.csv"), + avail_profile=resources("avail_profile_s{simpl}_{clusters}.csv"), + dsm_profile=resources("dsm_profile_s{simpl}_{clusters}.csv"), threads: 1 resources: mem_mb=2000, log: - LOGS + "build_transport_demand_s{simpl}_{clusters}.log", + logs("build_transport_demand_s{simpl}_{clusters}.log"), conda: "../envs/environment.yaml" script: @@ -727,57 +755,62 @@ rule prepare_sector_network: **build_biomass_transport_costs_output, **gas_infrastructure, **build_sequestration_potentials_output, - network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", - energy_totals_name=RESOURCES + "energy_totals.csv", + network=resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"), + energy_totals_name=resources("energy_totals.csv"), eurostat=input_eurostat, - pop_weighted_energy_totals=RESOURCES - + "pop_weighted_energy_totals_s{simpl}_{clusters}.csv", - shipping_demand=RESOURCES + "shipping_demand_s{simpl}_{clusters}.csv", - transport_demand=RESOURCES + "transport_demand_s{simpl}_{clusters}.csv", - transport_data=RESOURCES + "transport_data_s{simpl}_{clusters}.csv", - avail_profile=RESOURCES + "avail_profile_s{simpl}_{clusters}.csv", - dsm_profile=RESOURCES + "dsm_profile_s{simpl}_{clusters}.csv", - co2_totals_name=RESOURCES + "co2_totals.csv", + pop_weighted_energy_totals=resources( + "pop_weighted_energy_totals_s{simpl}_{clusters}.csv" + ), + shipping_demand=resources("shipping_demand_s{simpl}_{clusters}.csv"), + transport_demand=resources("transport_demand_s{simpl}_{clusters}.csv"), + transport_data=resources("transport_data_s{simpl}_{clusters}.csv"), + avail_profile=resources("avail_profile_s{simpl}_{clusters}.csv"), + dsm_profile=resources("dsm_profile_s{simpl}_{clusters}.csv"), + co2_totals_name=resources("co2_totals.csv"), co2="data/eea/UNFCCC_v23.csv", - biomass_potentials=RESOURCES + "biomass_potentials_s{simpl}_{clusters}.csv", + biomass_potentials=resources("biomass_potentials_s{simpl}_{clusters}.csv"), heat_profile="data/heat_load_profile_BDEW.csv", costs="data/costs_{}.csv".format(config["costs"]["year"]) if config["foresight"] == "overnight" else "data/costs_{planning_horizons}.csv", - profile_offwind_ac=RESOURCES + "profile_offwind-ac.nc", - profile_offwind_dc=RESOURCES + "profile_offwind-dc.nc", - h2_cavern=RESOURCES + "salt_cavern_potentials_s{simpl}_{clusters}.csv", - busmap_s=RESOURCES + "busmap_elec_s{simpl}.csv", - busmap=RESOURCES + "busmap_elec_s{simpl}_{clusters}.csv", - clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", - simplified_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}.csv", - industrial_demand=RESOURCES - + "industrial_energy_demand_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - heat_demand_urban=RESOURCES + "heat_demand_urban_elec_s{simpl}_{clusters}.nc", - heat_demand_rural=RESOURCES + "heat_demand_rural_elec_s{simpl}_{clusters}.nc", - heat_demand_total=RESOURCES + "heat_demand_total_elec_s{simpl}_{clusters}.nc", - temp_soil_total=RESOURCES + "temp_soil_total_elec_s{simpl}_{clusters}.nc", - temp_soil_rural=RESOURCES + "temp_soil_rural_elec_s{simpl}_{clusters}.nc", - temp_soil_urban=RESOURCES + "temp_soil_urban_elec_s{simpl}_{clusters}.nc", - temp_air_total=RESOURCES + "temp_air_total_elec_s{simpl}_{clusters}.nc", - temp_air_rural=RESOURCES + "temp_air_rural_elec_s{simpl}_{clusters}.nc", - temp_air_urban=RESOURCES + "temp_air_urban_elec_s{simpl}_{clusters}.nc", - cop_soil_total=RESOURCES + "cop_soil_total_elec_s{simpl}_{clusters}.nc", - cop_soil_rural=RESOURCES + "cop_soil_rural_elec_s{simpl}_{clusters}.nc", - cop_soil_urban=RESOURCES + "cop_soil_urban_elec_s{simpl}_{clusters}.nc", - cop_air_total=RESOURCES + "cop_air_total_elec_s{simpl}_{clusters}.nc", - cop_air_rural=RESOURCES + "cop_air_rural_elec_s{simpl}_{clusters}.nc", - cop_air_urban=RESOURCES + "cop_air_urban_elec_s{simpl}_{clusters}.nc", - solar_thermal_total=RESOURCES - + "solar_thermal_total_elec_s{simpl}_{clusters}.nc" + profile_offwind_ac=resources("profile_offwind-ac.nc"), + profile_offwind_dc=resources("profile_offwind-dc.nc"), + h2_cavern=resources("salt_cavern_potentials_s{simpl}_{clusters}.csv"), + busmap_s=resources("busmap_elec_s{simpl}.csv"), + busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"), + clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), + simplified_pop_layout=resources("pop_layout_elec_s{simpl}.csv"), + industrial_demand=resources( + "industrial_energy_demand_elec_s{simpl}_{clusters}_{planning_horizons}.csv" + ), + heat_demand_urban=resources("heat_demand_urban_elec_s{simpl}_{clusters}.nc"), + heat_demand_rural=resources("heat_demand_rural_elec_s{simpl}_{clusters}.nc"), + heat_demand_total=resources("heat_demand_total_elec_s{simpl}_{clusters}.nc"), + temp_soil_total=resources("temp_soil_total_elec_s{simpl}_{clusters}.nc"), + temp_soil_rural=resources("temp_soil_rural_elec_s{simpl}_{clusters}.nc"), + temp_soil_urban=resources("temp_soil_urban_elec_s{simpl}_{clusters}.nc"), + temp_air_total=resources("temp_air_total_elec_s{simpl}_{clusters}.nc"), + temp_air_rural=resources("temp_air_rural_elec_s{simpl}_{clusters}.nc"), + temp_air_urban=resources("temp_air_urban_elec_s{simpl}_{clusters}.nc"), + cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"), + cop_soil_rural=resources("cop_soil_rural_elec_s{simpl}_{clusters}.nc"), + cop_soil_urban=resources("cop_soil_urban_elec_s{simpl}_{clusters}.nc"), + cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"), + cop_air_rural=resources("cop_air_rural_elec_s{simpl}_{clusters}.nc"), + cop_air_urban=resources("cop_air_urban_elec_s{simpl}_{clusters}.nc"), + solar_thermal_total=resources( + "solar_thermal_total_elec_s{simpl}_{clusters}.nc" + ) if config["sector"]["solar_thermal"] else [], - solar_thermal_urban=RESOURCES - + "solar_thermal_urban_elec_s{simpl}_{clusters}.nc" + solar_thermal_urban=resources( + "solar_thermal_urban_elec_s{simpl}_{clusters}.nc" + ) if config["sector"]["solar_thermal"] else [], - solar_thermal_rural=RESOURCES - + "solar_thermal_rural_elec_s{simpl}_{clusters}.nc" + solar_thermal_rural=resources( + "solar_thermal_rural_elec_s{simpl}_{clusters}.nc" + ) if config["sector"]["solar_thermal"] else [], output: diff --git a/rules/collect.smk b/rules/collect.smk index 70dc4641..e0f19a4c 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -23,7 +23,7 @@ rule all: rule cluster_networks: input: expand( - RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", + resources("networks/elec_s{simpl}_{clusters}.nc"), **config["scenario"], run=config["run"]["name"] ), @@ -32,7 +32,7 @@ rule cluster_networks: rule extra_components_networks: input: expand( - RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", + resources("networks/elec_s{simpl}_{clusters}_ec.nc"), **config["scenario"], run=config["run"]["name"] ), @@ -41,7 +41,7 @@ rule extra_components_networks: rule prepare_elec_networks: input: expand( - RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"), **config["scenario"], run=config["run"]["name"] ), diff --git a/rules/common.smk b/rules/common.smk index f24301c8..0467d560 100644 --- a/rules/common.smk +++ b/rules/common.smk @@ -40,10 +40,12 @@ def static_getter(wildcards, keys, default): def dynamic_getter(wildcards, keys, default): """Getter function for dynamic config values based on scenario.""" + if "run" not in wildcards: + return get_config(config, keys, default) scenario_name = wildcards.run if scenario_name not in scenarios: raise ValueError( - f"Scenario {scenario_name} not found in file {config['scenariofile']}." + f"Scenario {scenario_name} not found in file {config['run']['scenario']['file']}." ) return get_config(scenario_config(scenario_name), keys, default) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 4c833b4f..e09103cc 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -14,7 +14,7 @@ rule plot_network: input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", + regions=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), output: map=RESULTS + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 34e2eb7c..1c454633 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -92,9 +92,9 @@ if config["enable"]["retrieve"] and config["enable"].get( static=True, ), output: - RESOURCES + "natura.tiff", + resources("natura.tiff"), log: - LOGS + "retrieve_natura_raster.log", + logs("retrieve_natura_raster.log"), resources: mem_mb=5000, retries: 2 diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index c396ebd5..cfdb1da0 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -12,7 +12,7 @@ rule solve_network: "co2_sequestration_potential", 200 ), input: - network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + network=resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"), config=RESULTS + "config.yaml", output: network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 8a93d24a..214733b7 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -12,13 +12,13 @@ rule add_existing_baseyear: input: network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - powerplants=RESOURCES + "powerplants.csv", - busmap_s=RESOURCES + "busmap_elec_s{simpl}.csv", - busmap=RESOURCES + "busmap_elec_s{simpl}_{clusters}.csv", - clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", + powerplants=resources("powerplants.csv"), + busmap_s=resources("busmap_elec_s{simpl}.csv"), + busmap=resources("busmap_elec_s{simpl}_{clusters}.csv"), + clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), costs="data/costs_{}.csv".format(config["scenario"]["planning_horizons"][0]), - cop_soil_total=RESOURCES + "cop_soil_total_elec_s{simpl}_{clusters}.nc", - cop_air_total=RESOURCES + "cop_air_total_elec_s{simpl}_{clusters}.nc", + cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"), + cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"), existing_heating="data/existing_infrastructure/existing_heating_raw.csv", existing_solar="data/existing_infrastructure/solar_capacity_IRENA.csv", existing_onwind="data/existing_infrastructure/onwind_capacity_IRENA.csv", @@ -55,8 +55,8 @@ rule add_brownfield: + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", network_p=solved_previous_horizon, #solved network at previous time step costs="data/costs_{planning_horizons}.csv", - cop_soil_total=RESOURCES + "cop_soil_total_elec_s{simpl}_{clusters}.nc", - cop_air_total=RESOURCES + "cop_air_total_elec_s{simpl}_{clusters}.nc", + cop_soil_total=resources("cop_soil_total_elec_s{simpl}_{clusters}.nc"), + cop_air_total=resources("cop_air_total_elec_s{simpl}_{clusters}.nc"), output: RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index c7700760..d8476868 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -31,8 +31,7 @@ rule solve_sector_network: walltime=config["solving"].get("walltime", "12:00:00"), benchmark: ( - RESULTS - + BENCHMARKS + BENCHMARKS + "solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" ) conda: diff --git a/rules/validate.smk b/rules/validate.smk index cfb8c959..09344673 100644 --- a/rules/validate.smk +++ b/rules/validate.smk @@ -20,9 +20,9 @@ rule build_electricity_production: snapshots=config["snapshots"], countries=config["countries"], output: - RESOURCES + "historical_electricity_production.csv", + resources("historical_electricity_production.csv"), log: - LOGS + "build_electricity_production.log", + logs("build_electricity_production.log"), resources: mem_mb=5000, script: @@ -38,11 +38,11 @@ rule build_cross_border_flows: snapshots=config["snapshots"], countries=config["countries"], input: - network=RESOURCES + "networks/base.nc", + network=resources("networks/base.nc"), output: - RESOURCES + "historical_cross_border_flows.csv", + resources("historical_cross_border_flows.csv"), log: - LOGS + "build_cross_border_flows.log", + logs("build_cross_border_flows.log"), resources: mem_mb=5000, script: @@ -58,9 +58,9 @@ rule build_electricity_prices: snapshots=config["snapshots"], countries=config["countries"], output: - RESOURCES + "historical_electricity_prices.csv", + resources("historical_electricity_prices.csv"), log: - LOGS + "build_electricity_prices.log", + logs("build_electricity_prices.log"), resources: mem_mb=5000, script: @@ -70,7 +70,7 @@ rule build_electricity_prices: rule plot_validation_electricity_production: input: network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", - electricity_production=RESOURCES + "historical_electricity_production.csv", + electricity_production=resources("historical_electricity_production.csv"), output: **{ plot: RESULTS @@ -88,7 +88,7 @@ rule plot_validation_cross_border_flows: countries=config["countries"], input: network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", - cross_border_flows=RESOURCES + "historical_cross_border_flows.csv", + cross_border_flows=resources("historical_cross_border_flows.csv"), output: **{ plot: RESULTS @@ -104,7 +104,7 @@ rule plot_validation_cross_border_flows: rule plot_validation_electricity_prices: input: network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", - electricity_prices=RESOURCES + "historical_electricity_prices.csv", + electricity_prices=resources("historical_electricity_prices.csv"), output: **{ plot: RESULTS diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..fc781c2f --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 3951be18..c166c61c 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -6,14 +6,13 @@ import contextlib import logging import os +import re import urllib from pathlib import Path import pandas as pd import pytz import yaml -from pypsa.components import component_attrs, components -from pypsa.descriptors import Dict from snakemake.utils import update_config from tqdm import tqdm @@ -22,6 +21,43 @@ logger = logging.getLogger(__name__) REGION_COLS = ["geometry", "name", "x", "y", "country"] +def path_provider(dir, rdir, shared_resources): + """ + Dynamically provide paths based on shared resources. + + Use this function whenever there is an input or output to a + snakemake rule that should, optionally, be either shared across runs + or created individually for each run. If shared_resources is a + string, it is assumed to be the wildcard that indicates the cutoff + after which resources are no longer shared. The function returns a + function which takes a filename and returns a path that is either + shared or individual to each run. + """ + + def path(fn): + pattern = r"\{([^{}]+)\}" + existing_wildcards = list(re.findall(pattern, fn)) + if shared_resources == "base": + # special case for shared "base" resources + no_relevant_wildcards = not len(set(existing_wildcards) - {"technology"}) + no_elec_rule = not fn.startswith("networks/elec") and not fn.startswith( + "add_electricity" + ) + is_shared = no_relevant_wildcards and no_elec_rule + elif isinstance(shared_resources, str): + final_wildcard = shared_resources + is_shared = final_wildcard not in existing_wildcards[:-1] + else: + is_shared = shared_resources + + if is_shared: + return f"{dir}{fn}" + else: + return f"{dir}{rdir}{fn}" + + return path + + # Define a context manager to temporarily mute print statements @contextlib.contextmanager def mute_print(): @@ -31,15 +67,16 @@ def mute_print(): def set_scenario_config(snakemake): - if snakemake.config["run"]["scenarios"] and "run" in snakemake.wildcards.keys(): + scenario = snakemake.config["run"].get("scenario", {}) + if scenario.get("enable") and "run" in snakemake.wildcards.keys(): try: - with open(snakemake.config["scenariofile"], "r") as f: + with open(scenario["file"], "r") as f: scenario_config = yaml.safe_load(f) except FileNotFoundError: # fallback for mock_snakemake script_dir = Path(__file__).parent.resolve() root_dir = script_dir.parent - with open(root_dir / snakemake.config["scenariofile"], "r") as f: + with open(root_dir / scenario["file"], "r") as f: scenario_config = yaml.safe_load(f) update_config(snakemake.config, scenario_config[snakemake.wildcards.run])