diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c2be3909..ff481b46 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -83,6 +83,7 @@ jobs: snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime snakemake -call all --configfile config/test/config.overnight.yaml --rerun-triggers=mtime snakemake -call all --configfile config/test/config.myopic.yaml --rerun-triggers=mtime + snakemake -call all --configfile config/test/config.electricity.scenario.yaml - name: Upload artifacts uses: actions/upload-artifact@v3 diff --git a/.gitignore b/.gitignore index 0adf0ae6..67cab0c9 100644 --- a/.gitignore +++ b/.gitignore @@ -19,10 +19,15 @@ gurobi.log /notebooks /data /cutouts - +/tmp doc/_build +/scripts/old +/scripts/create_scenarios.py + config.yaml +config/scenario.yaml + dconf /data/links_p_nom.csv @@ -51,25 +56,15 @@ publications.jrc.ec.europa.eu/ *.nc *~ -/scripts/old *.pyc -/cutouts -/tmp -/pypsa *.xlsx -config.yaml - -doc/_build - *.xls *.geojson *.ipynb -data/costs_* - merger-todos.md diff --git a/Snakefile b/Snakefile index 0e783beb..e495e7d3 100644 --- a/Snakefile +++ b/Snakefile @@ -4,14 +4,13 @@ from os.path import normpath, exists from shutil import copyfile, move, rmtree - +from pathlib import Path +import yaml from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider - -HTTP = HTTPRemoteProvider() - from snakemake.utils import min_version min_version("7.7") +HTTP = HTTPRemoteProvider() if not exists("config/config.yaml"): @@ -24,8 +23,16 @@ configfile: "config/config.yaml" COSTS = f"data/costs_{config['costs']['year']}.csv" ATLITE_NPROCESSES = config["atlite"].get("nprocesses", 4) -run = config.get("run", {}) -RDIR = run["name"] + "/" if run.get("name") else "" +run = config["run"] +if run.get("scenarios", False): + if run["shared_resources"]: + raise ValueError("Cannot use shared resources with scenarios") + scenarios = yaml.safe_load(Path(config["scenariofile"]).read_text()) + RDIR = "{run}/" +elif run["name"]: + RDIR = run["name"] + "/" +else: + RDIR = "" CDIR = RDIR if not run.get("shared_cutouts") else "" LOGS = "logs/" + RDIR diff --git a/config/config.default.yaml b/config/config.default.yaml index b162b75d..5357db8d 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -5,6 +5,7 @@ # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#top-level-configuration version: 0.8.1 tutorial: false +scenariofile: config/scenarios.yaml logging: level: INFO @@ -21,6 +22,7 @@ remote: # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#run run: name: "" + scenarios: false disable_progressbar: false shared_resources: false shared_cutouts: true diff --git a/config/create_scenarios.py b/config/create_scenarios.py deleted file mode 100644 index 9a5f9a98..00000000 --- a/config/create_scenarios.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors -# -# SPDX-License-Identifier: MIT - -import itertools - -# Insert your config values that should be altered in the template. -template = """ -scenario{scenario_number}: - sector: - carbon_: {config_value} - - config_section2: - config_key2: {config_value2} -""" - -# Define all possible combinations of config values. -# This must define all config values that are used in the template. -config_values = dict(config_values=["true", "false"], config_values2=[1, 2, 3, 4, 5]) - -combinations = [ - dict(zip(config_values.keys(), values)) - for values in itertools.product(*config_values.values()) -] - -# write the scenarios to a file -filename = "scenarios.yaml" -with open(filename, "w") as f: - for i, config in enumerate(combinations): - f.write(template.format(scenario_number=i, **config)) diff --git a/config/scenarios.yaml b/config/scenarios.yaml index e69de29b..37d32243 100644 --- a/config/scenarios.yaml +++ b/config/scenarios.yaml @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + +# This file is used to define the scenarios that are run by snakemake. Each entry on the first level is a scenario. Each scenario can contain configuration overrides with respect to the config/config.yaml settings. +# +# Example +# +# custom-scenario: # name of the scenario +# electricity: +# renewable_carriers: [wind, solar] # override the list of renewable carriers diff --git a/config/test/config.scenarios.electricity.yaml b/config/test/config.scenarios.electricity.yaml new file mode 100644 index 00000000..0e4ced04 --- /dev/null +++ b/config/test/config.scenarios.electricity.yaml @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC0-1.0 + +tutorial: true +scenariofile: "config/test/scenarios.electricity.yaml" + +run: + name: + - test-elec-no-offshore-wind + - test-elec-no-onshore-wind + scenarios: true + disable_progressbar: true + shared_resources: false # cannot be true if scenarios is true + shared_cutouts: true + +scenario: + clusters: + - 5 + opts: + - Co2L-24H + +countries: ['BE'] + +snapshots: + start: "2013-03-01" + end: "2013-03-08" + +electricity: + co2limit: 100.e+6 + + extendable_carriers: + Generator: [OCGT] + StorageUnit: [battery] + Store: [H2] + Link: [H2 pipeline] + + renewable_carriers: [solar, onwind, offwind-ac, offwind-dc] + + +atlite: + default_cutout: be-03-2013-era5 + cutouts: + be-03-2013-era5: + module: era5 + x: [4., 15.] + y: [46., 56.] + time: ["2013-03-01", "2013-03-08"] + +renewable: + onwind: + cutout: be-03-2013-era5 + offwind-ac: + cutout: be-03-2013-era5 + max_depth: false + offwind-dc: + cutout: be-03-2013-era5 + max_depth: false + solar: + cutout: be-03-2013-era5 + + +clustering: + exclude_carriers: ["OCGT", "offwind-ac", "coal"] + +lines: + dynamic_line_rating: + activate: true + cutout: be-03-2013-era5 + max_line_rating: 1.3 + + +solving: + solver: + name: glpk + options: "glpk-default" + + +plotting: + map: + boundaries: + eu_node_location: + x: -5.5 + y: 46. + costs_max: 1000 + costs_threshold: 0.0000001 + energy_max: + energy_min: + energy_threshold: 0.000001 diff --git a/config/test/scenarios.electricity.yaml b/config/test/scenarios.electricity.yaml new file mode 100644 index 00000000..e9893479 --- /dev/null +++ b/config/test/scenarios.electricity.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC0-1.0 + +test-elec-no-offshore-wind: + electricity: + renewable_carriers: [solar, onwind] + + +test-elec-no-onshore-wind: + electricity: + extendable_carriers: + Generator: [OCGT] + renewable_carriers: [solar, offwind-ac, offwind-dc] diff --git a/doc/configtables/run.csv b/doc/configtables/run.csv index 90cf65ad..2d5cf5d9 100644 --- a/doc/configtables/run.csv +++ b/doc/configtables/run.csv @@ -1,5 +1,6 @@ ,Unit,Values,Description -name,--,"any string","Specify a name for your run. Results will be stored under this name." +name,--,str/list,"Specify a name for your run. Results will be stored under this name. If ``scenarios`` is set to ``true``, the name must contain a subset of scenario names defined in ``scenariofile``." +scenarios,--,bool,"{true, false}","Switch to select whether workflow should generate scenarios based on ``scenariofile``." disable_progrssbar,bool,"{true, false}","Switch to select whether progressbar should be disabled." shared_resources,bool,"{true, false}","Switch to select whether resources should be shared across runs." shared_cutouts,bool,"{true, false}","Switch to select whether cutouts should be shared across runs." diff --git a/doc/configtables/toplevel.csv b/doc/configtables/toplevel.csv index 67954389..8cbb3e56 100644 --- a/doc/configtables/toplevel.csv +++ b/doc/configtables/toplevel.csv @@ -1,6 +1,7 @@ ,Unit,Values,Description version,--,0.x.x,Version of PyPSA-Eur. Descriptive only. tutorial,bool,"{true, false}",Switch to retrieve the tutorial data set instead of the full data set. +scenariofile,str,,Path to the scenario yaml file. The scenario file contains config overrides for each scenario. In order to be taken account, ``run:scenarios`` has to be set to ``true`` and ``run:name`` has to be a subset of top level keys given in the scenario file. In order to automatically create a `scenario.yaml` file based on a combindation of settings, alter and use the ``create_scenarios.py`` script in ``scripts``. logging,,, -- level,--,"Any of {'INFO', 'WARNING', 'ERROR'}","Restrict console outputs to all infos, warning or errors only" -- format,--,,Custom format for log messages. See `LogRecord `_ attributes. diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 383951bd..4d59c058 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -20,9 +20,9 @@ if config["enable"].get("prepare_links_p_nom", False): rule build_electricity_demand: params: - snapshots=config["snapshots"], - countries=config["countries"], - load=config["load"], + snapshots=config_provider("snapshots"), + countries=config_provider("countries"), + load=config_provider("load"), input: ancient("data/load_raw.csv"), output: @@ -39,9 +39,9 @@ rule build_electricity_demand: rule build_powerplants: params: - powerplants_filter=config["electricity"]["powerplants_filter"], - custom_powerplants=config["electricity"]["custom_powerplants"], - countries=config["countries"], + powerplants_filter=config_provider("electricity", "powerplants_filter"), + custom_powerplants=config_provider("electricity", "custom_powerplants"), + countries=config_provider("countries"), input: base_network=RESOURCES + "networks/base.nc", custom_powerplants="data/custom_powerplants.csv", @@ -60,11 +60,11 @@ rule build_powerplants: rule base_network: params: - countries=config["countries"], - snapshots=config["snapshots"], - lines=config["lines"], - links=config["links"], - transformers=config["transformers"], + countries=config_provider("countries"), + snapshots=config_provider("snapshots"), + lines=config_provider("lines"), + links=config_provider("links"), + transformers=config_provider("transformers"), input: eg_buses="data/entsoegridkit/buses.csv", eg_lines="data/entsoegridkit/lines.csv", @@ -94,7 +94,7 @@ rule base_network: rule build_shapes: params: - countries=config["countries"], + countries=config_provider("countries"), input: naturalearth=ancient("data/bundle/naturalearth/ne_10m_admin_0_countries.shp"), eez=ancient("data/bundle/eez/World_EEZ_v8_2014.shp"), @@ -121,7 +121,7 @@ rule build_shapes: rule build_bus_regions: params: - countries=config["countries"], + countries=config_provider("countries"), input: country_shapes=RESOURCES + "country_shapes.geojson", offshore_shapes=RESOURCES + "offshore_shapes.geojson", @@ -144,8 +144,8 @@ if config["enable"].get("build_cutout", False): rule build_cutout: params: - snapshots=config["snapshots"], - cutouts=config["atlite"]["cutouts"], + snapshots=config_provider("snapshots"), + cutouts=config_provider("atlite", "cutouts"), input: regions_onshore=RESOURCES + "regions_onshore.geojson", regions_offshore=RESOURCES + "regions_offshore.geojson", @@ -208,7 +208,7 @@ rule build_ship_raster: rule build_renewable_profiles: params: - renewable=config["renewable"], + renewable=config_provider("renewable"), input: base_network=RESOURCES + "networks/base.nc", corine=ancient("data/bundle/corine/g250_clc06_V18_5.tif"), @@ -277,8 +277,8 @@ rule build_monthly_prices: rule build_hydro_profile: params: - hydro=config["renewable"]["hydro"], - countries=config["countries"], + hydro=config_provider("renewable", "hydro"), + countries=config_provider("countries"), input: country_shapes=RESOURCES + "country_shapes.geojson", eia_hydro_generation="data/eia_hydro_annual_generation.csv", @@ -321,13 +321,13 @@ if config["lines"]["dynamic_line_rating"]["activate"]: rule add_electricity: params: - length_factor=config["lines"]["length_factor"], - scaling_factor=config["load"]["scaling_factor"], - countries=config["countries"], - renewable=config["renewable"], - electricity=config["electricity"], - conventional=config["conventional"], - costs=config["costs"], + length_factor=config_provider("lines", "length_factor"), + scaling_factor=config_provider("load", "scaling_factor"), + countries=config_provider("countries"), + renewable=config_provider("renewable"), + electricity=config_provider("electricity"), + conventional=config_provider("conventional"), + costs=config_provider("costs"), input: **{ f"profile_{tech}": RESOURCES + f"profile_{tech}.nc" @@ -370,14 +370,16 @@ rule add_electricity: rule simplify_network: params: - simplify_network=config["clustering"]["simplify_network"], - aggregation_strategies=config["clustering"].get("aggregation_strategies", {}), - focus_weights=config.get("focus_weights", None), - renewable_carriers=config["electricity"]["renewable_carriers"], - max_hours=config["electricity"]["max_hours"], - length_factor=config["lines"]["length_factor"], - p_max_pu=config["links"].get("p_max_pu", 1.0), - costs=config["costs"], + simplify_network=config_provider("clustering", "simplify_network"), + aggregation_strategies=config_provider( + "clustering", "aggregation_strategies", default={} + ), + focus_weights=config_provider("focus_weights", default=None), + renewable_carriers=config_provider("electricity", "renewable_carriers"), + max_hours=config_provider("electricity", "max_hours"), + length_factor=config_provider("lines", "length_factor"), + p_max_pu=config_provider("links", "p_max_pu", default=1.0), + costs=config_provider("costs"), input: network=RESOURCES + "networks/elec.nc", tech_costs=COSTS, @@ -404,15 +406,19 @@ rule simplify_network: rule cluster_network: params: - cluster_network=config["clustering"]["cluster_network"], - aggregation_strategies=config["clustering"].get("aggregation_strategies", {}), - custom_busmap=config["enable"].get("custom_busmap", False), - focus_weights=config.get("focus_weights", None), - renewable_carriers=config["electricity"]["renewable_carriers"], - conventional_carriers=config["electricity"].get("conventional_carriers", []), - max_hours=config["electricity"]["max_hours"], - length_factor=config["lines"]["length_factor"], - costs=config["costs"], + cluster_network=config_provider("clustering", "cluster_network"), + aggregation_strategies=config_provider( + "clustering", "aggregation_strategies", default={} + ), + custom_busmap=config_provider("enable", "custom_busmap", default=False), + focus_weights=config_provider("focus_weights", default=None), + renewable_carriers=config_provider("electricity", "renewable_carriers"), + conventional_carriers=config_provider( + "electricity", "conventional_carriers", default=[] + ), + max_hours=config_provider("electricity", "max_hours"), + length_factor=config_provider("lines", "length_factor"), + costs=config_provider("costs"), input: network=RESOURCES + "networks/elec_s{simpl}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson", @@ -445,9 +451,9 @@ rule cluster_network: rule add_extra_components: params: - extendable_carriers=config["electricity"]["extendable_carriers"], - max_hours=config["electricity"]["max_hours"], - costs=config["costs"], + extendable_carriers=config_provider("electricity", "extendable_carriers"), + max_hours=config_provider("electricity", "max_hours"), + costs=config_provider("costs"), input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", tech_costs=COSTS, @@ -468,13 +474,13 @@ rule add_extra_components: rule prepare_network: params: - links=config["links"], - lines=config["lines"], - co2base=config["electricity"]["co2base"], - co2limit=config["electricity"]["co2limit"], - gaslimit=config["electricity"].get("gaslimit"), - max_hours=config["electricity"]["max_hours"], - costs=config["costs"], + links=config_provider("links"), + lines=config_provider("lines"), + co2base=config_provider("electricity", "co2base"), + co2limit=config_provider("electricity", "co2limit"), + gaslimit=config_provider("electricity", "gaslimit"), + max_hours=config_provider("electricity", "max_hours"), + costs=config_provider("costs"), input: RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", tech_costs=COSTS, diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 356abdc5..cc80ab64 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -141,7 +141,7 @@ if not (config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]): rule build_heat_demands: params: - snapshots=config["snapshots"], + snapshots=config_provider("snapshots"), input: pop_layout=RESOURCES + "pop_layout_{scope}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -163,7 +163,7 @@ rule build_heat_demands: rule build_temperature_profiles: params: - snapshots=config["snapshots"], + snapshots=config_provider("snapshots"), input: pop_layout=RESOURCES + "pop_layout_{scope}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -186,7 +186,7 @@ rule build_temperature_profiles: rule build_cop_profiles: params: - heat_pump_sink_T=config["sector"]["heat_pump_sink_T"], + heat_pump_sink_T=config_provider("sector", "heat_pump_sink_T"), input: temp_soil_total=RESOURCES + "temp_soil_total_elec_s{simpl}_{clusters}.nc", temp_soil_rural=RESOURCES + "temp_soil_rural_elec_s{simpl}_{clusters}.nc", @@ -215,8 +215,8 @@ rule build_cop_profiles: rule build_solar_thermal_profiles: params: - snapshots=config["snapshots"], - solar_thermal=config["solar_thermal"], + snapshots=config_provider("snapshots"), + solar_thermal=config_provider("solar_thermal"), input: pop_layout=RESOURCES + "pop_layout_{scope}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -238,8 +238,8 @@ rule build_solar_thermal_profiles: rule build_energy_totals: params: - countries=config["countries"], - energy=config["energy"], + countries=config_provider("countries"), + energy=config_provider("energy"), input: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", co2="data/eea/UNFCCC_v23.csv", @@ -266,7 +266,7 @@ rule build_energy_totals: rule build_biomass_potentials: params: - biomass=config["biomass"], + biomass=config_provider("biomass"), input: enspreso_biomass=HTTP.remote( "https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx", @@ -329,9 +329,9 @@ if config["sector"]["regional_co2_sequestration_potential"]["enable"]: rule build_sequestration_potentials: params: - sequestration_potential=config["sector"][ - "regional_co2_sequestration_potential" - ], + sequestration_potential=config_provider( + "sector", "regional_co2_sequestration_potential" + ), input: sequestration_potential=HTTP.remote( "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson", @@ -386,7 +386,7 @@ rule build_salt_cavern_potentials: rule build_ammonia_production: params: - countries=config["countries"], + countries=config_provider("countries"), input: usgs="data/myb1-2017-nitro.xls", output: @@ -406,8 +406,8 @@ rule build_ammonia_production: rule build_industry_sector_ratios: params: - industry=config["industry"], - ammonia=config["sector"].get("ammonia", False), + industry=config_provider("industry"), + ammonia=config_provider("sector", "ammonia", default=False), input: ammonia_production=RESOURCES + "ammonia_production.csv", idees="data/jrc-idees-2015", @@ -428,8 +428,8 @@ rule build_industry_sector_ratios: rule build_industrial_production_per_country: params: - industry=config["industry"], - countries=config["countries"], + industry=config_provider("industry"), + countries=config_provider("countries"), input: ammonia_production=RESOURCES + "ammonia_production.csv", jrc="data/jrc-idees-2015", @@ -452,7 +452,7 @@ rule build_industrial_production_per_country: rule build_industrial_production_per_country_tomorrow: params: - industry=config["industry"], + industry=config_provider("industry"), input: industrial_production_per_country=RESOURCES + "industrial_production_per_country.csv", @@ -478,8 +478,10 @@ rule build_industrial_production_per_country_tomorrow: rule build_industrial_distribution_key: params: - hotmaps_locate_missing=config["industry"].get("hotmaps_locate_missing", False), - countries=config["countries"], + hotmaps_locate_missing=config_provider( + "industry", "hotmaps_locate_missing", default=False + ), + countries=config_provider("countries"), input: regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", @@ -555,8 +557,8 @@ rule build_industrial_energy_demand_per_node: rule build_industrial_energy_demand_per_country_today: params: - countries=config["countries"], - industry=config["industry"], + countries=config_provider("countries"), + industry=config_provider("industry"), input: jrc="data/jrc-idees-2015", ammonia_production=RESOURCES + "ammonia_production.csv", @@ -604,8 +606,8 @@ if config["sector"]["retrofitting"]["retro_endogen"]: rule build_retro_cost: params: - retrofitting=config["sector"]["retrofitting"], - countries=config["countries"], + retrofitting=config_provider("sector", "retrofitting"), + countries=config_provider("countries"), input: building_stock="data/retro/data_building_stock.csv", data_tabula="data/retro/tabula-calculator-calcsetbuilding.csv", @@ -677,8 +679,8 @@ rule build_shipping_demand: rule build_transport_demand: params: - snapshots=config["snapshots"], - sector=config["sector"], + snapshots=config_provider("snapshots"), + sector=config_provider("sector"), input: clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", pop_weighted_energy_totals=RESOURCES @@ -705,18 +707,20 @@ rule build_transport_demand: rule prepare_sector_network: params: - co2_budget=config["co2_budget"], - conventional_carriers=config["existing_capacities"]["conventional_carriers"], - foresight=config["foresight"], - costs=config["costs"], - sector=config["sector"], - industry=config["industry"], - pypsa_eur=config["pypsa_eur"], - length_factor=config["lines"]["length_factor"], - planning_horizons=config["scenario"]["planning_horizons"], - countries=config["countries"], - emissions_scope=config["energy"]["emissions"], - eurostat_report_year=config["energy"]["eurostat_report_year"], + co2_budget=config_provider("co2_budget"), + conventional_carriers=config_provider( + "existing_capacities", "conventional_carriers" + ), + foresight=config_provider("foresight"), + costs=config_provider("costs"), + sector=config_provider("sector"), + industry=config_provider("industry"), + pypsa_eur=config_provider("pypsa_eur"), + length_factor=config_provider("lines", "length_factor"), + planning_horizons=config_provider("scenario", "planning_horizons"), + countries=config_provider("countries"), + emissions_scope=config_provider("energy", "emissions"), + eurostat_report_year=config_provider("energy", "eurostat_report_year"), RDIR=RDIR, input: **build_retro_cost_output, diff --git a/rules/collect.smk b/rules/collect.smk index 74f26ccb..a29aa715 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -22,13 +22,19 @@ rule all: rule cluster_networks: input: - expand(RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", **config["scenario"]), + expand( + RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", + **config["scenario"], + run=config["run"]["name"] + ), rule extra_components_networks: input: expand( - RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", **config["scenario"] + RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", + **config["scenario"], + run=config["run"]["name"] ), @@ -36,7 +42,8 @@ rule prepare_elec_networks: input: expand( RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", - **config["scenario"] + **config["scenario"], + run=config["run"]["name"] ), @@ -45,7 +52,8 @@ rule prepare_sector_networks: expand( RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - **config["scenario"] + **config["scenario"], + run=config["run"]["name"] ), @@ -53,7 +61,8 @@ rule solve_elec_networks: input: expand( RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", - **config["scenario"] + **config["scenario"], + run=config["run"]["name"] ), @@ -62,7 +71,8 @@ rule solve_sector_networks: expand( RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - **config["scenario"] + **config["scenario"], + run=config["run"]["name"] ), @@ -71,7 +81,8 @@ rule plot_networks: expand( RESULTS + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", - **config["scenario"] + **config["scenario"], + run=config["run"]["name"] ), @@ -80,11 +91,13 @@ rule validate_elec_networks: expand( RESULTS + "figures/.statistics_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}", - **config["scenario"] + **config["scenario"], + run=config["run"]["name"] ), expand( RESULTS + "figures/.validation_{kind}_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}", **config["scenario"], + run=config["run"]["name"], kind=["production", "prices", "cross_border"] ), diff --git a/rules/common.smk b/rules/common.smk index ec5be355..5677f577 100644 --- a/rules/common.smk +++ b/rules/common.smk @@ -2,6 +2,57 @@ # # SPDX-License-Identifier: MIT +import copy + + +def get_config(keys, config, default=None): + """Retrieve a nested value from a dictionary using a tuple of keys.""" + value = config + for key in keys: + value = value.get(key, default) + if value == default: + return default + return value + + +def merge_configs(base_config, scenario_config): + """Merge base config with a specific scenario without modifying the original.""" + merged = copy.deepcopy(base_config) + for key, value in scenario_config.items(): + if key in merged and isinstance(merged[key], dict): + merged[key] = merge_configs(merged[key], value) + else: + merged[key] = value + return merged + + +def config_provider(*keys, default=None): + """Dynamically provide config values based on 'run' -> 'name'. + + Usage in Snakemake rules would look something like: + params: + my_param=config_provider("key1", "key2", default="some_default_value") + """ + + def static_getter(wildcards): + """Getter function for static config values.""" + return get_config(keys, config, default) + + def dynamic_getter(wildcards): + """Getter function for dynamic config values based on scenario.""" + scenario_name = wildcards.run + if scenario_name not in scenarios: + raise ValueError( + f"Scenario {scenario_name} not found in file {config['scenariofile']}." + ) + merged_config = merge_configs(config, scenarios[scenario_name]) + return get_config(keys, merged_config, default) + + if config["run"].get("scenarios", False): + return dynamic_getter + else: + return static_getter + def memory(w): factor = 3.0 diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 2618680e..fccda6e4 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -10,8 +10,8 @@ localrules: rule plot_network: params: - foresight=config["foresight"], - plotting=config["plotting"], + foresight=config_provider("foresight"), + plotting=config_provider("plotting"), input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -53,16 +53,17 @@ rule copy_config: rule make_summary: params: - foresight=config["foresight"], - costs=config["costs"], - snapshots=config["snapshots"], - scenario=config["scenario"], + foresight=config_provider("foresight"), + costs=config_provider("costs"), + snapshots=config_provider("snapshots"), + scenario=config_provider("scenario"), RDIR=RDIR, input: networks=expand( RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - **config["scenario"] + **config["scenario"], + run=config["run"]["name"] ), costs="data/costs_{}.csv".format(config["costs"]["year"]) if config["foresight"] == "overnight" @@ -70,7 +71,8 @@ rule make_summary: plots=expand( RESULTS + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", - **config["scenario"] + **config["scenario"], + run=config["run"]["name"] ), output: nodal_costs=RESULTS + "csvs/nodal_costs.csv", @@ -103,10 +105,10 @@ rule make_summary: rule plot_summary: params: - countries=config["countries"], - planning_horizons=config["scenario"]["planning_horizons"], - sector_opts=config["scenario"]["sector_opts"], - plotting=config["plotting"], + countries=config_provider("countries"), + planning_horizons=config_provider("scenario", "planning_horizons"), + sector_opts=config_provider("scenario", "sector_opts"), + plotting=config_provider("plotting"), RDIR=RDIR, input: costs=RESULTS + "csvs/costs.csv", @@ -145,7 +147,7 @@ STATISTICS_BARPLOTS = [ rule plot_elec_statistics: params: - plotting=config["plotting"], + plotting=config_provider("plotting"), barplots=STATISTICS_BARPLOTS, input: network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 0b60ee2e..4b9e9542 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -29,7 +29,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", output: expand("data/bundle/{file}", file=datafiles), log: - LOGS + "retrieve_databundle.log", + "logs/retrieve_databundle.log", resources: mem_mb=1000, retries: 2 @@ -72,7 +72,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", T output: "data/costs_{year}.csv", log: - LOGS + "retrieve_cost_data_{year}.log", + "logs/retrieve_cost_data_{year}.log", resources: mem_mb=1000, retries: 2 @@ -123,7 +123,7 @@ if config["enable"]["retrieve"] and config["enable"].get( output: *datafiles, log: - LOGS + "retrieve_sector_databundle.log", + "logs/retrieve_sector_databundle.log", retries: 2 conda: "../envs/environment.yaml" @@ -145,7 +145,7 @@ if config["enable"]["retrieve"] and ( output: expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles), log: - LOGS + "retrieve_gas_infrastructure_data.log", + "logs/retrieve_gas_infrastructure_data.log", retries: 2 conda: "../envs/environment.yaml" @@ -169,7 +169,7 @@ if config["enable"]["retrieve"]: output: "data/load_raw.csv", log: - LOGS + "retrieve_electricity_demand.log", + "logs/retrieve_electricity_demand.log", resources: mem_mb=5000, retries: 2 @@ -189,7 +189,7 @@ if config["enable"]["retrieve"]: output: "data/shipdensity_global.zip", log: - LOGS + "retrieve_ship_raster.log", + "logs/retrieve_ship_raster.log", resources: mem_mb=5000, retries: 2 @@ -209,7 +209,7 @@ if config["enable"]["retrieve"]: output: "data/validation/emission-spot-primary-market-auction-report-2019-data.xls", log: - LOGS + "retrieve_monthly_co2_prices.log", + "logs/retrieve_monthly_co2_prices.log", resources: mem_mb=5000, retries: 2 @@ -223,7 +223,7 @@ if config["enable"]["retrieve"]: output: "data/validation/energy-price-trends-xlsx-5619002.xlsx", log: - LOGS + "retrieve_monthly_fuel_prices.log", + "logs/retrieve_monthly_fuel_prices.log", resources: mem_mb=5000, retries: 2 diff --git a/scripts/_helpers.py b/scripts/_helpers.py index fc7bc9e0..c66d708e 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -14,6 +14,7 @@ import pytz import yaml from pypsa.components import component_attrs, components from pypsa.descriptors import Dict +from snakemake.utils import update_config from tqdm import tqdm logger = logging.getLogger(__name__) @@ -29,6 +30,13 @@ def mute_print(): yield +def set_scenario_config(snakemake): + if "scenario_config" in snakemake.input: + with open(snakemake.input.scenario_config, "r") as f: + scenario_config = yaml.safe_load(f) + update_config(snakemake.config, scenario_config) + + def configure_logging(snakemake, skip_handlers=False): """ Configure the basic behaviour for the logging module. diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 56375800..ff5e950e 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -92,7 +92,7 @@ import powerplantmatching as pm import pypsa import scipy.sparse as sparse import xarray as xr -from _helpers import configure_logging, update_p_nom_max +from _helpers import configure_logging, set_scenario_config, update_p_nom_max from powerplantmatching.export import map_country_bus from shapely.prepared import prep @@ -809,6 +809,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("add_electricity") configure_logging(snakemake) + set_scenario_config(snakemake) params = snakemake.params diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index e00e1e5f..9fe20066 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -55,7 +55,7 @@ import logging import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config from add_electricity import load_costs, sanitize_carriers idx = pd.IndexSlice @@ -231,6 +231,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("add_extra_components", simpl="", clusters=5) configure_logging(snakemake) + set_scenario_config(snakemake) n = pypsa.Network(snakemake.input.network) extendable_carriers = snakemake.params.extendable_carriers diff --git a/scripts/base_network.py b/scripts/base_network.py index b4ac1d8c..b5304109 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -77,7 +77,7 @@ import shapely import shapely.prepared import shapely.wkt import yaml -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config from scipy import spatial from scipy.sparse import csgraph from shapely.geometry import LineString, Point @@ -745,6 +745,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("base_network") configure_logging(snakemake) + set_scenario_config(snakemake) n = base_network( snakemake.input.eg_buses, diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index a6500bb0..76a57f5e 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -47,7 +47,7 @@ import geopandas as gpd import numpy as np import pandas as pd import pypsa -from _helpers import REGION_COLS, configure_logging +from _helpers import REGION_COLS, configure_logging, set_scenario_config from scipy.spatial import Voronoi from shapely.geometry import Polygon @@ -115,6 +115,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_bus_regions") configure_logging(snakemake) + set_scenario_config(snakemake) countries = snakemake.params.countries diff --git a/scripts/build_cross_border_flows.py b/scripts/build_cross_border_flows.py index b9fc3fe8..743f1742 100644 --- a/scripts/build_cross_border_flows.py +++ b/scripts/build_cross_border_flows.py @@ -8,7 +8,7 @@ import logging import pandas as pd import pypsa -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config from entsoe import EntsoePandasClient from entsoe.exceptions import InvalidBusinessParameterError, NoMatchingDataError from requests import HTTPError @@ -21,6 +21,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_cross_border_flows") configure_logging(snakemake) + set_scenario_config(snakemake) api_key = snakemake.config["private"]["keys"]["entsoe_api"] client = EntsoePandasClient(api_key=api_key) diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 9a7f9e00..f9f951b5 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -95,7 +95,7 @@ import logging import atlite import geopandas as gpd import pandas as pd -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config logger = logging.getLogger(__name__) @@ -105,6 +105,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_cutout", cutout="europe-2013-era5") configure_logging(snakemake) + set_scenario_config(snakemake) cutout_params = snakemake.params.cutouts[snakemake.wildcards.cutout] diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py index 38c75544..60d40e1e 100755 --- a/scripts/build_electricity_demand.py +++ b/scripts/build_electricity_demand.py @@ -45,7 +45,7 @@ logger = logging.getLogger(__name__) import dateutil import numpy as np import pandas as pd -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config from pandas import Timedelta as Delta @@ -288,6 +288,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_electricity_demand") configure_logging(snakemake) + set_scenario_config(snakemake) powerstatistics = snakemake.params.load["power_statistics"] interpolate_limit = snakemake.params.load["interpolate_limit"] diff --git a/scripts/build_electricity_prices.py b/scripts/build_electricity_prices.py index 353ea7e3..48361afe 100644 --- a/scripts/build_electricity_prices.py +++ b/scripts/build_electricity_prices.py @@ -7,7 +7,7 @@ import logging import pandas as pd -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config from entsoe import EntsoePandasClient from entsoe.exceptions import NoMatchingDataError @@ -19,6 +19,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_cross_border_flows") configure_logging(snakemake) + set_scenario_config(snakemake) api_key = snakemake.config["private"]["keys"]["entsoe_api"] client = EntsoePandasClient(api_key=api_key) diff --git a/scripts/build_electricity_production.py b/scripts/build_electricity_production.py index beb859bd..38be2ba0 100644 --- a/scripts/build_electricity_production.py +++ b/scripts/build_electricity_production.py @@ -7,7 +7,7 @@ import logging import pandas as pd -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config from entsoe import EntsoePandasClient from entsoe.exceptions import NoMatchingDataError @@ -39,6 +39,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_electricity_production") configure_logging(snakemake) + set_scenario_config(snakemake) api_key = snakemake.config["private"]["keys"]["entsoe_api"] client = EntsoePandasClient(api_key=api_key) diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index bed666f2..883f33d2 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -65,7 +65,7 @@ import atlite import country_converter as coco import geopandas as gpd import pandas as pd -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config cc = coco.CountryConverter() @@ -129,6 +129,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_hydro_profile") configure_logging(snakemake) + set_scenario_config(snakemake) params_hydro = snakemake.params.hydro cutout = atlite.Cutout(snakemake.input.cutout) diff --git a/scripts/build_line_rating.py b/scripts/build_line_rating.py index 7f842d43..abc6b286 100755 --- a/scripts/build_line_rating.py +++ b/scripts/build_line_rating.py @@ -59,7 +59,7 @@ import numpy as np import pandas as pd import pypsa import xarray as xr -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config from shapely.geometry import LineString as Line from shapely.geometry import Point @@ -147,6 +147,7 @@ if __name__ == "__main__": opts="Co2L-4H", ) configure_logging(snakemake) + set_scenario_config(snakemake) n = pypsa.Network(snakemake.input.base_network) time = pd.date_range(freq="h", **snakemake.config["snapshots"]) diff --git a/scripts/build_monthly_prices.py b/scripts/build_monthly_prices.py index c2e88972..89edde79 100644 --- a/scripts/build_monthly_prices.py +++ b/scripts/build_monthly_prices.py @@ -46,7 +46,7 @@ Data was accessed at 16.5.2023 import logging import pandas as pd -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config logger = logging.getLogger(__name__) @@ -114,6 +114,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_monthly_prices") configure_logging(snakemake) + set_scenario_config(snakemake) fuel_price = get_fuel_price() fuel_price.to_csv(snakemake.output.fuel_price) diff --git a/scripts/build_natura_raster.py b/scripts/build_natura_raster.py index 8fdb4ea3..79418d5c 100644 --- a/scripts/build_natura_raster.py +++ b/scripts/build_natura_raster.py @@ -46,7 +46,7 @@ import logging import atlite import geopandas as gpd import rasterio as rio -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config from rasterio.features import geometry_mask from rasterio.warp import transform_bounds @@ -92,6 +92,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_natura_raster") configure_logging(snakemake) + set_scenario_config(snakemake) cutouts = snakemake.input.cutouts xs, Xs, ys, Ys = zip(*(determine_cutout_xXyY(cutout) for cutout in cutouts)) diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index cbe94505..2ad1e010 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -80,7 +80,7 @@ import logging import pandas as pd import powerplantmatching as pm import pypsa -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config from powerplantmatching.export import map_country_bus logger = logging.getLogger(__name__) @@ -115,6 +115,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_powerplants") configure_logging(snakemake) + set_scenario_config(snakemake) n = pypsa.Network(snakemake.input.base_network) countries = snakemake.params.countries diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 7b08325b..40b3151d 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -188,7 +188,7 @@ import geopandas as gpd import numpy as np import pandas as pd import xarray as xr -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config from dask.distributed import Client from pypsa.geo import haversine from shapely.geometry import LineString @@ -202,6 +202,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_renewable_profiles", technology="solar") configure_logging(snakemake) + set_scenario_config(snakemake) nprocesses = int(snakemake.threads) noprogress = snakemake.config["run"].get("disable_progressbar", True) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index eb837409..571a7282 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -77,7 +77,7 @@ import geopandas as gpd import numpy as np import pandas as pd import pycountry as pyc -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config from shapely.geometry import MultiPolygon, Polygon logger = logging.getLogger(__name__) @@ -254,6 +254,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_shapes") configure_logging(snakemake) + set_scenario_config(snakemake) country_shapes = countries(snakemake.input.naturalearth, snakemake.params.countries) country_shapes.reset_index().to_file(snakemake.output.country_shapes) diff --git a/scripts/build_ship_raster.py b/scripts/build_ship_raster.py index 90e006b0..25bebcca 100644 --- a/scripts/build_ship_raster.py +++ b/scripts/build_ship_raster.py @@ -44,9 +44,10 @@ Description import logging import os import zipfile +from pathlib import Path import rioxarray -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config from build_natura_raster import determine_cutout_xXyY logger = logging.getLogger(__name__) @@ -57,16 +58,19 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_ship_raster") configure_logging(snakemake) + set_scenario_config(snakemake) cutouts = snakemake.input.cutouts xs, Xs, ys, Ys = zip(*(determine_cutout_xXyY(cutout) for cutout in cutouts)) with zipfile.ZipFile(snakemake.input.ship_density) as zip_f: - zip_f.extract("shipdensity_global.tif") - with rioxarray.open_rasterio("shipdensity_global.tif") as ship_density: - ship_density = ship_density.drop(["band"]).sel( - x=slice(min(xs), max(Xs)), y=slice(max(Ys), min(ys)) - ) - ship_density.rio.to_raster(snakemake.output[0]) + resources = Path(snakemake.output[0]).parent + fn = "shipdensity_global.tif" + zip_f.extract(fn, resources) + with rioxarray.open_rasterio(resources / fn) as ship_density: + ship_density = ship_density.drop(["band"]).sel( + x=slice(min(xs), max(Xs)), y=slice(max(Ys), min(ys)) + ) + ship_density.rio.to_raster(snakemake.output[0]) - os.remove("shipdensity_global.tif") + (resources / fn).unlink() diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 884b6a2b..b0ce4796 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -133,7 +133,7 @@ import pandas as pd import pyomo.environ as po import pypsa import seaborn as sns -from _helpers import configure_logging, update_p_nom_max +from _helpers import configure_logging, set_scenario_config, update_p_nom_max from pypsa.clustering.spatial import ( busmap_by_greedy_modularity, busmap_by_hac, @@ -463,6 +463,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("cluster_network", simpl="", clusters="37") configure_logging(snakemake) + set_scenario_config(snakemake) params = snakemake.params solver_name = snakemake.config["solving"]["solver"]["name"] diff --git a/scripts/plot_statistics.py b/scripts/plot_statistics.py index 1e75203f..a0a3e71d 100644 --- a/scripts/plot_statistics.py +++ b/scripts/plot_statistics.py @@ -7,7 +7,7 @@ import matplotlib.pyplot as plt import pypsa import seaborn as sns -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config sns.set_theme("paper", style="whitegrid") @@ -24,6 +24,7 @@ if __name__ == "__main__": ll="v1.0", ) configure_logging(snakemake) + set_scenario_config(snakemake) n = pypsa.Network(snakemake.input.network) diff --git a/scripts/plot_validation_cross_border_flows.py b/scripts/plot_validation_cross_border_flows.py index 43ed45e9..8b063d8c 100644 --- a/scripts/plot_validation_cross_border_flows.py +++ b/scripts/plot_validation_cross_border_flows.py @@ -9,7 +9,7 @@ import matplotlib.pyplot as plt import pandas as pd import pypsa import seaborn as sns -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config sns.set_theme("paper", style="whitegrid") @@ -195,6 +195,7 @@ if __name__ == "__main__": ll="v1.0", ) configure_logging(snakemake) + set_scenario_config(snakemake) countries = snakemake.params.countries diff --git a/scripts/plot_validation_electricity_prices.py b/scripts/plot_validation_electricity_prices.py index 2a187b9f..c229e382 100644 --- a/scripts/plot_validation_electricity_prices.py +++ b/scripts/plot_validation_electricity_prices.py @@ -8,7 +8,7 @@ import matplotlib.pyplot as plt import pandas as pd import pypsa import seaborn as sns -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config from pypsa.statistics import get_bus_and_carrier sns.set_theme("paper", style="whitegrid") @@ -25,6 +25,7 @@ if __name__ == "__main__": ll="v1.0", ) configure_logging(snakemake) + set_scenario_config(snakemake) n = pypsa.Network(snakemake.input.network) n.loads.carrier = "load" diff --git a/scripts/plot_validation_electricity_production.py b/scripts/plot_validation_electricity_production.py index 5c5569d0..3e81faff 100644 --- a/scripts/plot_validation_electricity_production.py +++ b/scripts/plot_validation_electricity_production.py @@ -8,7 +8,7 @@ import matplotlib.pyplot as plt import pandas as pd import pypsa import seaborn as sns -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config from pypsa.statistics import get_bus_and_carrier sns.set_theme("paper", style="whitegrid") @@ -35,6 +35,7 @@ if __name__ == "__main__": ll="v1.0", ) configure_logging(snakemake) + set_scenario_config(snakemake) n = pypsa.Network(snakemake.input.network) n.loads.carrier = "load" diff --git a/scripts/prepare_links_p_nom.py b/scripts/prepare_links_p_nom.py index 4b915d22..450f3227 100644 --- a/scripts/prepare_links_p_nom.py +++ b/scripts/prepare_links_p_nom.py @@ -40,7 +40,7 @@ Description import logging import pandas as pd -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config logger = logging.getLogger(__name__) @@ -69,6 +69,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("prepare_links_p_nom", simpl="") configure_logging(snakemake) + set_scenario_config(snakemake) links_p_nom = pd.read_html( "https://en.wikipedia.org/wiki/List_of_HVDC_projects", header=0, match="SwePol" diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index a5a00a3c..a7f1ddf3 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -63,7 +63,7 @@ import re import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging +from _helpers import configure_logging, set_scenario_config from add_electricity import load_costs, update_transmission_costs from pypsa.descriptors import expand_series @@ -283,6 +283,7 @@ if __name__ == "__main__": "prepare_network", simpl="", clusters="37", ll="v1.0", opts="Ept" ) configure_logging(snakemake) + set_scenario_config(snakemake) opts = snakemake.wildcards.opts.split("-") diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index 75d8519e..cb3bdc11 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -36,7 +36,7 @@ import logging import tarfile from pathlib import Path -from _helpers import configure_logging, progress_retrieve +from _helpers import configure_logging, progress_retrieve, set_scenario_config logger = logging.getLogger(__name__) diff --git a/scripts/retrieve_monthly_fuel_prices.py b/scripts/retrieve_monthly_fuel_prices.py index 11e351ce..887014cc 100644 --- a/scripts/retrieve_monthly_fuel_prices.py +++ b/scripts/retrieve_monthly_fuel_prices.py @@ -12,7 +12,7 @@ logger = logging.getLogger(__name__) from pathlib import Path -from _helpers import configure_logging, progress_retrieve +from _helpers import configure_logging, progress_retrieve, set_scenario_config if __name__ == "__main__": if "snakemake" not in globals(): @@ -23,6 +23,7 @@ if __name__ == "__main__": else: rootpath = "." configure_logging(snakemake) + set_scenario_config(snakemake) url = "https://www.destatis.de/EN/Themes/Economy/Prices/Publications/Downloads-Energy-Price-Trends/energy-price-trends-xlsx-5619002.xlsx?__blob=publicationFile" diff --git a/scripts/retrieve_sector_databundle.py b/scripts/retrieve_sector_databundle.py index 0d172c8d..1beed478 100644 --- a/scripts/retrieve_sector_databundle.py +++ b/scripts/retrieve_sector_databundle.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) import tarfile from pathlib import Path -from _helpers import configure_logging, progress_retrieve +from _helpers import configure_logging, progress_retrieve, set_scenario_config if __name__ == "__main__": if "snakemake" not in globals(): @@ -24,6 +24,7 @@ if __name__ == "__main__": else: rootpath = "." configure_logging(snakemake) + set_scenario_config(snakemake) url = "https://zenodo.org/record/5824485/files/pypsa-eur-sec-data-bundle.tar.gz" diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index cac25647..440145ff 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -92,7 +92,7 @@ import numpy as np import pandas as pd import pypsa import scipy as sp -from _helpers import configure_logging, update_p_nom_max +from _helpers import configure_logging, set_scenario_config, update_p_nom_max from add_electricity import load_costs from cluster_network import cluster_regions, clustering_for_n_clusters from pypsa.clustering.spatial import ( @@ -531,6 +531,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("simplify_network", simpl="") configure_logging(snakemake) + set_scenario_config(snakemake) params = snakemake.params solver_name = snakemake.config["solving"]["solver"]["name"] diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 8eccef19..37b05286 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -33,7 +33,11 @@ import numpy as np import pandas as pd import pypsa import xarray as xr -from _helpers import configure_logging, update_config_with_sector_opts +from _helpers import ( + configure_logging, + set_scenario_config, + update_config_with_sector_opts, +) logger = logging.getLogger(__name__) pypsa.pf.logger.setLevel(logging.WARNING) @@ -657,6 +661,7 @@ if __name__ == "__main__": planning_horizons="2020", ) configure_logging(snakemake) + set_scenario_config(snakemake) if "sector_opts" in snakemake.wildcards.keys(): update_config_with_sector_opts( snakemake.config, snakemake.wildcards.sector_opts diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 1a3855a9..064d735a 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -11,7 +11,11 @@ import logging import numpy as np import pypsa -from _helpers import configure_logging, update_config_with_sector_opts +from _helpers import ( + configure_logging, + set_scenario_config, + update_config_with_sector_opts, +) from solve_network import prepare_network, solve_network logger = logging.getLogger(__name__) @@ -33,6 +37,7 @@ if __name__ == "__main__": ) configure_logging(snakemake) + set_scenario_config(snakemake) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) opts = (snakemake.wildcards.opts + "-" + snakemake.wildcards.sector_opts).split("-")