From 1758076815049e5e38b52859ba81132c21c446d2 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Thu, 11 May 2023 20:34:11 +0200 Subject: [PATCH 01/41] test param in add_electricity --- rules/build_electricity.smk | 8 ++++++ scripts/add_electricity.py | 50 ++++++++++++++++++------------------- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 133ceb6e..194e186d 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -273,6 +273,14 @@ rule add_electricity: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", output: RESOURCES + "networks/elec.nc", + params: + costs=snakemake.config["costs"] + electricity=snakemake.config["electricity"] + renewable=snakemake.config["renewable"] + conventional=snakemake.config.get("conventional", {}) + countries=snakemake.config["countries"] + scaling_factor=snakemake.config["load"]["scaling_factor"] + length_factor=snakemake.config["lines"]["length_factor"] log: LOGS + "add_electricity.log", benchmark: diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 1d32bce1..bed5ee6b 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -686,15 +686,15 @@ def estimate_renewable_capacities(n, config): ) -def add_nice_carrier_names(n, config): +def add_nice_carrier_names(n, plotting): carrier_i = n.carriers.index nice_names = ( - pd.Series(config["plotting"]["nice_names"]) + pd.Series(plotting["nice_names"]) .reindex(carrier_i) .fillna(carrier_i.to_series().str.title()) ) n.carriers["nice_name"] = nice_names - colors = pd.Series(config["plotting"]["tech_colors"]).reindex(carrier_i) + colors = pd.Series(plotting["tech_colors"]).reindex(carrier_i) if colors.isna().any(): missing_i = list(colors.index[colors.isna()]) logger.warning(f"tech_colors for carriers {missing_i} not defined in config.") @@ -713,23 +713,23 @@ if __name__ == "__main__": costs = load_costs( snakemake.input.tech_costs, - snakemake.config["costs"], - snakemake.config["electricity"], + snakemake.param["costs"], + snakemake.param["electricity"], Nyears, ) ppl = load_powerplants(snakemake.input.powerplants) - if "renewable_carriers" in snakemake.config["electricity"]: - renewable_carriers = set(snakemake.config["electricity"]["renewable_carriers"]) + if "renewable_carriers" in snakemake.param["electricity"]: + renewable_carriers = set(snakemake.param["electricity"]["renewable_carriers"]) else: logger.warning( "Missing key `renewable_carriers` under config entry `electricity`. " "In future versions, this will raise an error. " "Falling back to carriers listed under `renewable`." ) - renewable_carriers = snakemake.config["renewable"] + renewable_carriers = snakemake.param["renewable"] - extendable_carriers = snakemake.config["electricity"]["extendable_carriers"] + extendable_carriers = snakemake.param["electricity"]["extendable_carriers"] if not (set(renewable_carriers) & set(extendable_carriers["Generator"])): logger.warning( "No renewables found in config entry `extendable_carriers`. " @@ -737,18 +737,18 @@ if __name__ == "__main__": "Falling back to all renewables." ) - conventional_carriers = snakemake.config["electricity"]["conventional_carriers"] + conventional_carriers = snakemake.param["electricity"]["conventional_carriers"] attach_load( n, snakemake.input.regions, snakemake.input.load, snakemake.input.nuts3_shapes, - snakemake.config["countries"], - snakemake.config["load"]["scaling_factor"], + snakemake.param["countries"], + snakemake.param["load"]["scaling_factor"], ) - update_transmission_costs(n, costs, snakemake.config["lines"]["length_factor"]) + update_transmission_costs(n, costs, snakemake.param["lines"]["length_factor"]) conventional_inputs = { k: v for k, v in snakemake.input.items() if k.startswith("conventional_") @@ -759,7 +759,7 @@ if __name__ == "__main__": ppl, conventional_carriers, extendable_carriers, - snakemake.config.get("conventional", {}), + snakemake.param.get("conventional", {}), conventional_inputs, ) @@ -769,11 +769,11 @@ if __name__ == "__main__": snakemake.input, renewable_carriers, extendable_carriers, - snakemake.config["lines"]["length_factor"], + snakemake.param["lines"]["length_factor"], ) if "hydro" in renewable_carriers: - conf = snakemake.config["renewable"]["hydro"] + conf = snakemake.param["renewable"]["hydro"] attach_hydro( n, costs, @@ -784,7 +784,7 @@ if __name__ == "__main__": **conf, ) - if "estimate_renewable_capacities" not in snakemake.config["electricity"]: + if "estimate_renewable_capacities" not in snakemake.param["electricity"]: logger.warning( "Missing key `estimate_renewable_capacities` under config entry `electricity`. " "In future versions, this will raise an error. " @@ -792,18 +792,18 @@ if __name__ == "__main__": ) if ( "estimate_renewable_capacities_from_capacity_stats" - in snakemake.config["electricity"] + in snakemake.param["electricity"] ): estimate_renewable_caps = { "enable": True, - **snakemake.config["electricity"][ + **snakemake.param["electricity"][ "estimate_renewable_capacities_from_capacity_stats" ], } else: estimate_renewable_caps = {"enable": False} else: - estimate_renewable_caps = snakemake.config["electricity"][ + estimate_renewable_caps = snakemake.param["electricity"][ "estimate_renewable_capacities" ] if "enable" not in estimate_renewable_caps: @@ -819,21 +819,21 @@ if __name__ == "__main__": "Falling back to whether `renewable_capacities_from_opsd` is non-empty." ) from_opsd = bool( - snakemake.config["electricity"].get("renewable_capacities_from_opsd", False) + snakemake.param["electricity"].get("renewable_capacities_from_opsd", False) ) estimate_renewable_caps["from_opsd"] = from_opsd if estimate_renewable_caps["enable"]: if estimate_renewable_caps["from_opsd"]: - tech_map = snakemake.config["electricity"]["estimate_renewable_capacities"][ + tech_map = snakemake.param["electricity"]["estimate_renewable_capacities"][ "technology_mapping" ] attach_OPSD_renewables(n, tech_map) - estimate_renewable_capacities(n, snakemake.config) + estimate_renewable_capacities(n, snakemake.param) update_p_nom_max(n) - add_nice_carrier_names(n, snakemake.config) + add_nice_carrier_names(n, snakemake.param["plotting"]) - n.meta = snakemake.config + n.meta = snakemake.param n.export_to_netcdf(snakemake.output[0]) From aa50ea44cc550433b8db38bb241c359b8ccc92ce Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Thu, 11 May 2023 20:59:37 +0200 Subject: [PATCH 02/41] fix add_electricity mistake --- rules/build_electricity.smk | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 194e186d..8b7780db 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -274,13 +274,13 @@ rule add_electricity: output: RESOURCES + "networks/elec.nc", params: - costs=snakemake.config["costs"] - electricity=snakemake.config["electricity"] - renewable=snakemake.config["renewable"] - conventional=snakemake.config.get("conventional", {}) - countries=snakemake.config["countries"] - scaling_factor=snakemake.config["load"]["scaling_factor"] - length_factor=snakemake.config["lines"]["length_factor"] + costs=snakemake.config["costs"], + electricity=snakemake.config["electricity"], + renewable=snakemake.config["renewable"], + conventional=snakemake.config.get("conventional", {}), + countries=snakemake.config["countries"], + scaling_factor=snakemake.config["load"]["scaling_factor"], + length_factor=snakemake.config["lines"]["length_factor"], log: LOGS + "add_electricity.log", benchmark: From 5be21dfc5a4cefd5e01bbf7798663d18eeacc638 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 12 May 2023 09:07:16 +0200 Subject: [PATCH 03/41] add option for piecewise linear transmission loss approximation --- config/config.default.yaml | 2 +- doc/configtables/solving.csv | 2 +- doc/release_notes.rst | 4 ++++ envs/environment.yaml | 2 +- scripts/solve_network.py | 3 +++ 5 files changed, 10 insertions(+), 3 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 10e6a6ed..78c1385c 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -623,9 +623,9 @@ clustering: solving: #tmpdir: "path/to/tmp" options: - formulation: kirchhoff clip_p_max_pu: 1.e-2 load_shedding: false + transmission_losses: 0 noisy_costs: true skip_iterations: true track_iterations: false diff --git a/doc/configtables/solving.csv b/doc/configtables/solving.csv index cba28cbe..c252ff32 100644 --- a/doc/configtables/solving.csv +++ b/doc/configtables/solving.csv @@ -1,7 +1,7 @@ ,Unit,Values,Description options,,, --- formulation,--,"Any of {'angles', 'kirchhoff', 'cycles', 'ptdf'}","Specifies which variant of linearized power flow formulations to use in the optimisation problem. Recommended is 'kirchhoff'. Explained in `this article `_." -- load_shedding,bool/float,"{'true','false', float}","Add generators with very high marginal cost to simulate load shedding and avoid problem infeasibilities. If load shedding is a float, it denotes the marginal cost in EUR/kWh." +-- transmission_losses,int,"[0-9]","Add piecewise linear approximation of transmission losses based on n tangents. Defaults to 0, which means losses are ignored." -- noisy_costs,bool,"{'true','false'}","Add random noise to marginal cost of generators by :math:`\mathcal{U}(0.009,0,011)` and capital cost of lines and links by :math:`\mathcal{U}(0.09,0,11)`." -- min_iterations,--,int,"Minimum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run." -- max_iterations,--,int,"Maximum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run." diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 14f2939d..3af16477 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -25,6 +25,10 @@ Upcoming Release * Remove ``vresutils`` dependency. +* Add option to include a piecewise linear approximation of transmission losses, + e.g. by setting ``solving: options: transmission_losses: 2`` for an + approximation with two tangents. + PyPSA-Eur 0.8.0 (18th March 2023) ================================= diff --git a/envs/environment.yaml b/envs/environment.yaml index f970c9ba..9d800fdc 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -10,7 +10,7 @@ dependencies: - python>=3.8 - pip -- pypsa>=0.21.3 +- pypsa>=0.23 - atlite>=0.2.9 - dask diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 8f4fdf5b..e671ffd3 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -600,6 +600,7 @@ def solve_network(n, config, opts="", **kwargs): track_iterations = cf_solving.get("track_iterations", False) min_iterations = cf_solving.get("min_iterations", 4) max_iterations = cf_solving.get("max_iterations", 6) + transmission_losses = cf_solving.get("transmission_losses", 0) # add to network for extra_functionality n.config = config @@ -613,6 +614,7 @@ def solve_network(n, config, opts="", **kwargs): if skip_iterations: status, condition = n.optimize( solver_name=solver_name, + transmission_losses=transmission_losses, extra_functionality=extra_functionality, **solver_options, **kwargs, @@ -623,6 +625,7 @@ def solve_network(n, config, opts="", **kwargs): track_iterations=track_iterations, min_iterations=min_iterations, max_iterations=max_iterations, + transmission_losses=transmission_losses, extra_functionality=extra_functionality, **solver_options, **kwargs, From 67ef38b35eb2e46d8dcc5252216b73ee062465f0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 12 May 2023 13:59:41 +0200 Subject: [PATCH 04/41] retrieve*: handle extraction path via snakemake.output --- scripts/retrieve_databundle.py | 3 +-- scripts/retrieve_gas_infrastructure_data.py | 2 +- scripts/retrieve_sector_databundle.py | 16 +++++++++------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index 0c6a7feb..75d8519e 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -58,9 +58,8 @@ if __name__ == "__main__": else: url = "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz" - # Save locations tarball_fn = Path(f"{rootpath}/bundle.tar.xz") - to_fn = Path(f"{rootpath}/data") + to_fn = Path(rootpath) / Path(snakemake.output[0]).parent.parent logger.info(f"Downloading databundle from '{url}'.") disable_progress = snakemake.config["run"].get("disable_progressbar", False) diff --git a/scripts/retrieve_gas_infrastructure_data.py b/scripts/retrieve_gas_infrastructure_data.py index dda7bd8c..42b726db 100644 --- a/scripts/retrieve_gas_infrastructure_data.py +++ b/scripts/retrieve_gas_infrastructure_data.py @@ -29,7 +29,7 @@ if __name__ == "__main__": # Save locations zip_fn = Path(f"{rootpath}/IGGIELGN.zip") - to_fn = Path(f"{rootpath}/data/gas_network/scigrid-gas") + to_fn = Path(rootpath) / Path(snakemake.output[0]).parent.parent logger.info(f"Downloading databundle from '{url}'.") disable_progress = snakemake.config["run"].get("disable_progressbar", False) diff --git a/scripts/retrieve_sector_databundle.py b/scripts/retrieve_sector_databundle.py index 97426ab2..0991bbe3 100644 --- a/scripts/retrieve_sector_databundle.py +++ b/scripts/retrieve_sector_databundle.py @@ -10,23 +10,25 @@ import logging logger = logging.getLogger(__name__) -import os -import sys import tarfile from pathlib import Path -# Add pypsa-eur scripts to path for import of _helpers -sys.path.insert(0, os.getcwd() + "/../pypsa-eur/scripts") - from _helpers import configure_logging, progress_retrieve if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake("retrieve_databundle") + rootpath = ".." + else: + rootpath = "." configure_logging(snakemake) url = "https://zenodo.org/record/5824485/files/pypsa-eur-sec-data-bundle.tar.gz" - tarball_fn = Path("sector-bundle.tar.gz") - to_fn = Path("data") + tarball_fn = Path(f"{rootpath}/sector-bundle.tar.gz") + to_fn = Path(rootpath) / Path(snakemake.output[0]).parent logger.info(f"Downloading databundle from '{url}'.") disable_progress = snakemake.config["run"].get("disable_progressbar", False) From b2216355f10bf11147cc7f03e74b17f6dc4b60e6 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 12 May 2023 14:10:04 +0200 Subject: [PATCH 05/41] retrieve_sector_data: add another .parent --- scripts/retrieve_sector_databundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/retrieve_sector_databundle.py b/scripts/retrieve_sector_databundle.py index 0991bbe3..0d172c8d 100644 --- a/scripts/retrieve_sector_databundle.py +++ b/scripts/retrieve_sector_databundle.py @@ -28,7 +28,7 @@ if __name__ == "__main__": url = "https://zenodo.org/record/5824485/files/pypsa-eur-sec-data-bundle.tar.gz" tarball_fn = Path(f"{rootpath}/sector-bundle.tar.gz") - to_fn = Path(rootpath) / Path(snakemake.output[0]).parent + to_fn = Path(rootpath) / Path(snakemake.output[0]).parent.parent logger.info(f"Downloading databundle from '{url}'.") disable_progress = snakemake.config["run"].get("disable_progressbar", False) From 8f91963e7623d6cf574c0fd267b80b581f9560d8 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 12 May 2023 14:50:58 +0200 Subject: [PATCH 06/41] properly reference p_max_pu files --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index f910dee4..85ab35e2 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -418,7 +418,7 @@ def attach_conventional_generators( if f"conventional_{carrier}_{attr}" in conventional_inputs: # Values affecting generators of technology k country-specific # First map generator buses to countries; then map countries to p_max_pu - values = pd.read_csv(values, index_col=0).iloc[:, 0] + values = pd.read_csv(snakemake.input[f"conventional_{carrier}_{attr}"], index_col=0).iloc[:, 0] bus_values = n.buses.country.map(values) n.generators[attr].update( n.generators.loc[idx].bus.map(bus_values).dropna() From 38bae672da42db241eb885eceeb3a6097644eeb5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 May 2023 12:51:39 +0000 Subject: [PATCH 07/41] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_electricity.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 85ab35e2..4afa3e22 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -418,7 +418,9 @@ def attach_conventional_generators( if f"conventional_{carrier}_{attr}" in conventional_inputs: # Values affecting generators of technology k country-specific # First map generator buses to countries; then map countries to p_max_pu - values = pd.read_csv(snakemake.input[f"conventional_{carrier}_{attr}"], index_col=0).iloc[:, 0] + values = pd.read_csv( + snakemake.input[f"conventional_{carrier}_{attr}"], index_col=0 + ).iloc[:, 0] bus_values = n.buses.country.map(values) n.generators[attr].update( n.generators.loc[idx].bus.map(bus_values).dropna() From 1fc48d8753f93b2b2d2f24c2516cf16777d72cb4 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Mon, 15 May 2023 10:33:17 +0200 Subject: [PATCH 08/41] using python algorithm to edit rules and script --- rules/build_electricity.smk | 55 +++++++++++++--- rules/build_sector.smk | 65 +++++++++++++++++++ rules/postprocess.smk | 13 ++++ rules/retrieve.smk | 2 + rules/solve_electricity.smk | 4 ++ rules/solve_myopic.smk | 12 ++++ rules/solve_overnight.smk | 2 + scripts/_helpers.py | 2 +- scripts/add_brownfield.py | 8 +-- scripts/add_electricity.py | 50 +++++++------- scripts/add_existing_baseyear.py | 20 +++--- scripts/add_extra_components.py | 4 +- scripts/build_ammonia_production.py | 2 +- scripts/build_biomass_potentials.py | 2 +- scripts/build_bus_regions.py | 2 +- scripts/build_cop_profiles.py | 2 +- scripts/build_cutout.py | 4 +- scripts/build_electricity_demand.py | 12 ++-- scripts/build_energy_totals.py | 12 ++-- scripts/build_gas_input_locations.py | 2 +- scripts/build_gas_network.py | 2 +- scripts/build_heat_demand.py | 2 +- scripts/build_hydro_profile.py | 4 +- scripts/build_industrial_distribution_key.py | 6 +- ...ustrial_energy_demand_per_country_today.py | 6 +- ...build_industrial_production_per_country.py | 10 +-- ...ustrial_production_per_country_tomorrow.py | 2 +- scripts/build_industry_sector_ratios.py | 4 +- scripts/build_population_layouts.py | 2 +- scripts/build_powerplants.py | 6 +- scripts/build_renewable_profiles.py | 4 +- scripts/build_retro_cost.py | 4 +- scripts/build_sequestration_potentials.py | 2 +- scripts/build_shapes.py | 4 +- scripts/build_solar_thermal_profiles.py | 4 +- scripts/build_temperature_profiles.py | 2 +- scripts/build_transport_demand.py | 4 +- scripts/cluster_gas_network.py | 2 +- scripts/cluster_network.py | 18 ++--- scripts/make_summary.py | 22 +++---- scripts/plot_network.py | 16 ++--- scripts/plot_summary.py | 32 ++++----- scripts/prepare_network.py | 20 +++--- scripts/prepare_sector_network.py | 42 ++++++------ scripts/retrieve_databundle.py | 4 +- scripts/simplify_network.py | 8 +-- scripts/solve_network.py | 2 +- scripts/solve_operations_network.py | 2 +- 48 files changed, 324 insertions(+), 187 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 8b7780db..bb000f2b 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -19,6 +19,10 @@ if config["enable"].get("prepare_links_p_nom", False): rule build_electricity_demand: + params: + snapshots=config["snapshots"], + countries=config["countries"], + load=config["load"], input: ancient("data/load_raw.csv"), output: @@ -34,6 +38,9 @@ rule build_electricity_demand: rule build_powerplants: + params: + electricity=config["electricity"], + countries=config["countries"], input: base_network=RESOURCES + "networks/base.nc", custom_powerplants="data/custom_powerplants.csv", @@ -79,6 +86,8 @@ rule base_network: rule build_shapes: + params: + countries=config["countries"], input: naturalearth=ancient("data/bundle/naturalearth/ne_10m_admin_0_countries.shp"), eez=ancient("data/bundle/eez/World_EEZ_v8_2014.shp"), @@ -104,6 +113,8 @@ rule build_shapes: rule build_bus_regions: + params: + countries=config["countries"], input: country_shapes=RESOURCES + "country_shapes.geojson", offshore_shapes=RESOURCES + "offshore_shapes.geojson", @@ -125,6 +136,9 @@ rule build_bus_regions: if config["enable"].get("build_cutout", False): rule build_cutout: + params: + snapshots=config["snapshots"], + atlite=config["atlite"], input: regions_onshore=RESOURCES + "regions_onshore.geojson", regions_offshore=RESOURCES + "regions_offshore.geojson", @@ -186,6 +200,8 @@ rule build_ship_raster: rule build_renewable_profiles: + params: + renewable=config["renewable"], input: base_network=RESOURCES + "networks/base.nc", corine=ancient("data/bundle/corine/g250_clc06_V18_5.tif"), @@ -235,6 +251,9 @@ rule build_renewable_profiles: rule build_hydro_profile: + params: + countries=config["countries"], + renewable=config["renewable"], input: country_shapes=RESOURCES + "country_shapes.geojson", eia_hydro_generation="data/eia_hydro_annual_generation.csv", @@ -252,6 +271,13 @@ rule build_hydro_profile: rule add_electricity: + params: + lines=config["lines"], + load=config["load"], + countries=config["countries"], + renewable=config["renewable"], + electricity=config["electricity"], + costs=config["costs"], input: **{ f"profile_{tech}": RESOURCES + f"profile_{tech}.nc" @@ -273,14 +299,6 @@ rule add_electricity: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", output: RESOURCES + "networks/elec.nc", - params: - costs=snakemake.config["costs"], - electricity=snakemake.config["electricity"], - renewable=snakemake.config["renewable"], - conventional=snakemake.config.get("conventional", {}), - countries=snakemake.config["countries"], - scaling_factor=snakemake.config["load"]["scaling_factor"], - length_factor=snakemake.config["lines"]["length_factor"], log: LOGS + "add_electricity.log", benchmark: @@ -295,6 +313,10 @@ rule add_electricity: rule simplify_network: + params: + clustering=config["clustering"], + electricity=config["electricity"], + costs=config["costs"], input: network=RESOURCES + "networks/elec.nc", tech_costs=COSTS, @@ -320,6 +342,14 @@ rule simplify_network: rule cluster_network: + params: + solving=config["solving"], + electricity=config["electricity"], + costs=config["costs"], + lines=config["lines"], + renewable=config["renewable"], + clustering=config["clustering"], + enable=config["enable"], input: network=RESOURCES + "networks/elec_s{simpl}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson", @@ -351,6 +381,9 @@ rule cluster_network: rule add_extra_components: + params: + costs=config["costs"], + electricity=config["electricity"], input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", tech_costs=COSTS, @@ -370,6 +403,12 @@ rule add_extra_components: rule prepare_network: + params: + links=config["links"], + solving=config["solving"], + lines=config["lines"], + electricity=config["electricity"], + costs=config["costs"], input: RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", tech_costs=COSTS, diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 1b724d1a..d375b7b9 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -4,6 +4,8 @@ rule build_population_layouts: + params: + logging=config["logging"], input: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", urban_percent="data/urban_percent.csv", @@ -70,6 +72,8 @@ rule build_simplified_population_layouts: if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: rule build_gas_network: + params: + logging=config["logging"], input: gas_network="data/gas_network/scigrid-gas/data/IGGIELGN_PipeSegments.geojson", output: @@ -84,6 +88,8 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: "../scripts/build_gas_network.py" rule build_gas_input_locations: + params: + logging=config["logging"], input: lng=HTTP.remote( "https://globalenergymonitor.org/wp-content/uploads/2022/09/Europe-Gas-Tracker-August-2022.xlsx", @@ -110,6 +116,8 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: "../scripts/build_gas_input_locations.py" rule cluster_gas_network: + params: + logging=config["logging"], input: cleaned_gas_network=RESOURCES + "gas_network.csv", regions_onshore=RESOURCES @@ -140,6 +148,8 @@ if not (config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]): rule build_heat_demands: + params: + snapshots=config["snapshots"], input: pop_layout=RESOURCES + "pop_layout_{scope}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -160,6 +170,8 @@ rule build_heat_demands: rule build_temperature_profiles: + params: + snapshots=config["snapshots"], input: pop_layout=RESOURCES + "pop_layout_{scope}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -181,6 +193,8 @@ rule build_temperature_profiles: rule build_cop_profiles: + params: + sector=config["sector"], input: temp_soil_total=RESOURCES + "temp_soil_total_elec_s{simpl}_{clusters}.nc", temp_soil_rural=RESOURCES + "temp_soil_rural_elec_s{simpl}_{clusters}.nc", @@ -208,6 +222,9 @@ rule build_cop_profiles: rule build_solar_thermal_profiles: + params: + snapshots=config["snapshots"], + solar_thermal=config["solar_thermal"], input: pop_layout=RESOURCES + "pop_layout_{scope}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -228,6 +245,11 @@ rule build_solar_thermal_profiles: rule build_energy_totals: + params: + run=config["run"], + countries=config["countries"], + energy=config["energy"], + logging=config["logging"], input: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", co2="data/eea/UNFCCC_v23.csv", @@ -253,6 +275,8 @@ rule build_energy_totals: rule build_biomass_potentials: + params: + biomass=config["biomass"], input: enspreso_biomass=HTTP.remote( "https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx", @@ -315,6 +339,8 @@ if not config["sector"]["biomass_transport"]: if config["sector"]["regional_co2_sequestration_potential"]["enable"]: rule build_sequestration_potentials: + params: + sector=config["sector"], input: sequestration_potential=HTTP.remote( "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson", @@ -368,6 +394,8 @@ rule build_salt_cavern_potentials: rule build_ammonia_production: + params: + countries=config["countries"], input: usgs="data/myb1-2017-nitro.xls", output: @@ -386,6 +414,9 @@ rule build_ammonia_production: rule build_industry_sector_ratios: + params: + industry=config["industry"], + sector=config["sector"], input: ammonia_production=RESOURCES + "ammonia_production.csv", idees="data/jrc-idees-2015", @@ -405,6 +436,11 @@ rule build_industry_sector_ratios: rule build_industrial_production_per_country: + params: + run=config["run"], + industry=config["industry"], + countries=config["countries"], + logging=config["logging"], input: ammonia_production=RESOURCES + "ammonia_production.csv", jrc="data/jrc-idees-2015", @@ -426,6 +462,8 @@ rule build_industrial_production_per_country: rule build_industrial_production_per_country_tomorrow: + params: + industry=config["industry"], input: industrial_production_per_country=RESOURCES + "industrial_production_per_country.csv", @@ -450,6 +488,10 @@ rule build_industrial_production_per_country_tomorrow: rule build_industrial_distribution_key: + params: + industry=config["industry"], + countries=config["countries"], + logging=config["logging"], input: regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", @@ -524,6 +566,10 @@ rule build_industrial_energy_demand_per_node: rule build_industrial_energy_demand_per_country_today: + params: + run=config["run"], + countries=config["countries"], + industry=config["industry"], input: jrc="data/jrc-idees-2015", ammonia_production=RESOURCES + "ammonia_production.csv", @@ -570,6 +616,9 @@ rule build_industrial_energy_demand_per_node_today: if config["sector"]["retrofitting"]["retro_endogen"]: rule build_retro_cost: + params: + sector=config["sector"], + countries=config["countries"], input: building_stock="data/retro/data_building_stock.csv", data_tabula="data/retro/tabula-calculator-calcsetbuilding.csv", @@ -640,6 +689,9 @@ rule build_shipping_demand: rule build_transport_demand: + params: + snapshots=config["snapshots"], + sector=config["sector"], input: clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", pop_weighted_energy_totals=RESOURCES @@ -666,6 +718,19 @@ rule build_transport_demand: rule prepare_sector_network: params: + co2_budget=config["co2_budget"], + solving=config["solving"], + existing_capacities=config["existing_capacities"], + foresight=config["foresight"], + costs=config["costs"], + logging=config["logging"], + sector=config["sector"], + industry=config["industry"], + pypsa_eur=config["pypsa_eur"], + lines=config["lines"], + scenario=config["scenario"], + countries=config["countries"], + energy=config["energy"], RDIR=RDIR, input: **build_retro_cost_output, diff --git a/rules/postprocess.smk b/rules/postprocess.smk index fae0f856..1dfdd098 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -9,6 +9,10 @@ localrules: rule plot_network: + params: + logging=config["logging"], + foresight=config["foresight"], + plotting=config["plotting"], input: overrides="data/override_component_attrs", network=RESULTS @@ -67,6 +71,11 @@ rule copy_conda_env: rule make_summary: params: + foresight=config["foresight"], + costs=config["costs"], + snapshots=config["snapshots"], + logging=config["logging"], + scenario=config["scenario"], RDIR=RDIR, input: overrides="data/override_component_attrs", @@ -114,6 +123,10 @@ rule make_summary: rule plot_summary: params: + logging=config["logging"], + countries=config["countries"], + scenario=config["scenario"], + plotting=config["plotting"], RDIR=RDIR, input: costs=RESULTS + "csvs/costs.csv", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 0a96406a..4bfbd6c6 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -19,6 +19,8 @@ if config["enable"].get("retrieve_databundle", True): datafiles.extend(["natura/Natura2000_end2015.shp", "GEBCO_2014_2D.nc"]) rule retrieve_databundle: + params: + tutorial=config["tutorial"], output: expand("data/bundle/{file}", file=datafiles), log: diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 8ddeca92..400220c1 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -4,6 +4,8 @@ rule solve_network: + params: + solving=config["solving"], input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: @@ -30,6 +32,8 @@ rule solve_network: rule solve_operations_network: + params: + solving=config["solving"], input: network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index f10d8157..2ecba999 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -4,6 +4,12 @@ rule add_existing_baseyear: + params: + scenario=config["scenario"], + sector=config["sector"], + logging=config["logging"], + existing_capacities=config["existing_capacities"], + costs=config["costs"], input: overrides="data/override_component_attrs", network=RESULTS @@ -42,6 +48,10 @@ rule add_existing_baseyear: rule add_brownfield: + params: + logging=config["logging"], + sector=config["sector"], + existing_capacities=config["existing_capacities"], input: overrides="data/override_component_attrs", network=RESULTS @@ -74,6 +84,8 @@ ruleorder: add_existing_baseyear > add_brownfield rule solve_sector_network_myopic: + params: + solving=config["solving"], input: overrides="data/override_component_attrs", network=RESULTS diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index c2e103e5..c3608471 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -4,6 +4,8 @@ rule solve_sector_network: + params: + solving=config["solving"], input: overrides="data/override_component_attrs", network=RESULTS diff --git a/scripts/_helpers.py b/scripts/_helpers.py index a67fb105..2cc9a05a 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -82,7 +82,7 @@ def load_network(import_name=None, custom_components=None): As in pypsa.Network(import_name) custom_components : dict Dictionary listing custom components. - For using ``snakemake.config['override_components']`` + For using ``snakemake.params['override_components']`` in ``config/config.yaml`` define: .. code:: yaml diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index e08b86d7..1d4e3a80 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -49,7 +49,7 @@ def add_brownfield(n, n_p, year): ) ] - threshold = snakemake.config["existing_capacities"]["threshold_capacity"] + threshold = snakemake.params["existing_capacities"]["threshold_capacity"] if not chp_heat.empty: threshold_chp_heat = ( @@ -87,7 +87,7 @@ def add_brownfield(n, n_p, year): # deal with gas network pipe_carrier = ["gas pipeline"] - if snakemake.config["sector"]["H2_retrofit"]: + if snakemake.params["sector"]["H2_retrofit"]: # drop capacities of previous year to avoid duplicating to_drop = n.links.carrier.isin(pipe_carrier) & (n.links.build_year != year) n.mremove("Link", n.links.loc[to_drop].index) @@ -98,7 +98,7 @@ def add_brownfield(n, n_p, year): & (n.links.build_year != year) ].index gas_pipes_i = n.links[n.links.carrier.isin(pipe_carrier)].index - CH4_per_H2 = 1 / snakemake.config["sector"]["H2_retrofit_capacity_per_CH4"] + CH4_per_H2 = 1 / snakemake.params["sector"]["H2_retrofit_capacity_per_CH4"] fr = "H2 pipeline retrofitted" to = "gas pipeline" # today's pipe capacity @@ -139,7 +139,7 @@ if __name__ == "__main__": planning_horizons=2030, ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index bed5ee6b..c3e847ce 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -686,15 +686,15 @@ def estimate_renewable_capacities(n, config): ) -def add_nice_carrier_names(n, plotting): +def add_nice_carrier_names(n, config): carrier_i = n.carriers.index nice_names = ( - pd.Series(plotting["nice_names"]) + pd.Series(config["plotting"]["nice_names"]) .reindex(carrier_i) .fillna(carrier_i.to_series().str.title()) ) n.carriers["nice_name"] = nice_names - colors = pd.Series(plotting["tech_colors"]).reindex(carrier_i) + colors = pd.Series(config["plotting"]["tech_colors"]).reindex(carrier_i) if colors.isna().any(): missing_i = list(colors.index[colors.isna()]) logger.warning(f"tech_colors for carriers {missing_i} not defined in config.") @@ -713,23 +713,23 @@ if __name__ == "__main__": costs = load_costs( snakemake.input.tech_costs, - snakemake.param["costs"], - snakemake.param["electricity"], + snakemake.params["costs"], + snakemake.params["electricity"], Nyears, ) ppl = load_powerplants(snakemake.input.powerplants) - if "renewable_carriers" in snakemake.param["electricity"]: - renewable_carriers = set(snakemake.param["electricity"]["renewable_carriers"]) + if "renewable_carriers" in snakemake.params["electricity"]: + renewable_carriers = set(snakemake.params["electricity"]["renewable_carriers"]) else: logger.warning( "Missing key `renewable_carriers` under config entry `electricity`. " "In future versions, this will raise an error. " "Falling back to carriers listed under `renewable`." ) - renewable_carriers = snakemake.param["renewable"] + renewable_carriers = snakemake.params["renewable"] - extendable_carriers = snakemake.param["electricity"]["extendable_carriers"] + extendable_carriers = snakemake.params["electricity"]["extendable_carriers"] if not (set(renewable_carriers) & set(extendable_carriers["Generator"])): logger.warning( "No renewables found in config entry `extendable_carriers`. " @@ -737,18 +737,18 @@ if __name__ == "__main__": "Falling back to all renewables." ) - conventional_carriers = snakemake.param["electricity"]["conventional_carriers"] + conventional_carriers = snakemake.params["electricity"]["conventional_carriers"] attach_load( n, snakemake.input.regions, snakemake.input.load, snakemake.input.nuts3_shapes, - snakemake.param["countries"], - snakemake.param["load"]["scaling_factor"], + snakemake.params["countries"], + snakemake.params["load"]["scaling_factor"], ) - update_transmission_costs(n, costs, snakemake.param["lines"]["length_factor"]) + update_transmission_costs(n, costs, snakemake.params["lines"]["length_factor"]) conventional_inputs = { k: v for k, v in snakemake.input.items() if k.startswith("conventional_") @@ -759,7 +759,7 @@ if __name__ == "__main__": ppl, conventional_carriers, extendable_carriers, - snakemake.param.get("conventional", {}), + snakemake.config.get("conventional", {}), conventional_inputs, ) @@ -769,11 +769,11 @@ if __name__ == "__main__": snakemake.input, renewable_carriers, extendable_carriers, - snakemake.param["lines"]["length_factor"], + snakemake.params["lines"]["length_factor"], ) if "hydro" in renewable_carriers: - conf = snakemake.param["renewable"]["hydro"] + conf = snakemake.params["renewable"]["hydro"] attach_hydro( n, costs, @@ -784,7 +784,7 @@ if __name__ == "__main__": **conf, ) - if "estimate_renewable_capacities" not in snakemake.param["electricity"]: + if "estimate_renewable_capacities" not in snakemake.params["electricity"]: logger.warning( "Missing key `estimate_renewable_capacities` under config entry `electricity`. " "In future versions, this will raise an error. " @@ -792,18 +792,18 @@ if __name__ == "__main__": ) if ( "estimate_renewable_capacities_from_capacity_stats" - in snakemake.param["electricity"] + in snakemake.params["electricity"] ): estimate_renewable_caps = { "enable": True, - **snakemake.param["electricity"][ + **snakemake.params["electricity"][ "estimate_renewable_capacities_from_capacity_stats" ], } else: estimate_renewable_caps = {"enable": False} else: - estimate_renewable_caps = snakemake.param["electricity"][ + estimate_renewable_caps = snakemake.params["electricity"][ "estimate_renewable_capacities" ] if "enable" not in estimate_renewable_caps: @@ -819,21 +819,21 @@ if __name__ == "__main__": "Falling back to whether `renewable_capacities_from_opsd` is non-empty." ) from_opsd = bool( - snakemake.param["electricity"].get("renewable_capacities_from_opsd", False) + snakemake.params["electricity"].get("renewable_capacities_from_opsd", False) ) estimate_renewable_caps["from_opsd"] = from_opsd if estimate_renewable_caps["enable"]: if estimate_renewable_caps["from_opsd"]: - tech_map = snakemake.param["electricity"]["estimate_renewable_capacities"][ + tech_map = snakemake.params["electricity"]["estimate_renewable_capacities"][ "technology_mapping" ] attach_OPSD_renewables(n, tech_map) - estimate_renewable_capacities(n, snakemake.param) + estimate_renewable_capacities(n, snakemake.config) update_p_nom_max(n) - add_nice_carrier_names(n, snakemake.param["plotting"]) + add_nice_carrier_names(n, snakemake.config) - n.meta = snakemake.param + n.meta = snakemake.config n.export_to_netcdf(snakemake.output[0]) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 5bc0960d..a24b078d 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -157,7 +157,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas # Fill missing DateOut dateout = ( df_agg.loc[biomass_i, "DateIn"] - + snakemake.config["costs"]["fill_values"]["lifetime"] + + snakemake.params["costs"]["fill_values"]["lifetime"] ) df_agg.loc[biomass_i, "DateOut"] = df_agg.loc[biomass_i, "DateOut"].fillna(dateout) @@ -218,7 +218,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas capacity = df.loc[grouping_year, generator] capacity = capacity[~capacity.isna()] capacity = capacity[ - capacity > snakemake.config["existing_capacities"]["threshold_capacity"] + capacity > snakemake.params["existing_capacities"]["threshold_capacity"] ] suffix = "-ac" if generator == "offwind" else "" name_suffix = f" {generator}{suffix}-{grouping_year}" @@ -582,7 +582,7 @@ def add_heating_capacities_installed_before_baseyear( ) # delete links with capacities below threshold - threshold = snakemake.config["existing_capacities"]["threshold_capacity"] + threshold = snakemake.params["existing_capacities"]["threshold_capacity"] n.mremove( "Link", [ @@ -608,14 +608,14 @@ if __name__ == "__main__": planning_horizons=2020, ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) - options = snakemake.config["sector"] + options = snakemake.params["sector"] opts = snakemake.wildcards.sector_opts.split("-") - baseyear = snakemake.config["scenario"]["planning_horizons"][0] + baseyear = snakemake.params["scenario"]["planning_horizons"][0] overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) @@ -626,14 +626,14 @@ if __name__ == "__main__": Nyears = n.snapshot_weightings.generators.sum() / 8760.0 costs = prepare_costs( snakemake.input.costs, - snakemake.config["costs"], + snakemake.params["costs"], Nyears, ) - grouping_years_power = snakemake.config["existing_capacities"][ + grouping_years_power = snakemake.params["existing_capacities"][ "grouping_years_power" ] - grouping_years_heat = snakemake.config["existing_capacities"]["grouping_years_heat"] + grouping_years_heat = snakemake.params["existing_capacities"]["grouping_years_heat"] add_power_capacities_installed_before_baseyear( n, grouping_years_power, costs, baseyear ) @@ -650,7 +650,7 @@ if __name__ == "__main__": .to_pandas() .reindex(index=n.snapshots) ) - default_lifetime = snakemake.config["costs"]["fill_values"]["lifetime"] + default_lifetime = snakemake.params["costs"]["fill_values"]["lifetime"] add_heating_capacities_installed_before_baseyear( n, baseyear, diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index 020370e5..08178c0a 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -235,11 +235,11 @@ if __name__ == "__main__": configure_logging(snakemake) n = pypsa.Network(snakemake.input.network) - elec_config = snakemake.config["electricity"] + elec_config = snakemake.params["electricity"] Nyears = n.snapshot_weightings.objective.sum() / 8760.0 costs = load_costs( - snakemake.input.tech_costs, snakemake.config["costs"], elec_config, Nyears + snakemake.input.tech_costs, snakemake.params["costs"], elec_config, Nyears ) attach_storageunits(n, costs, elec_config) diff --git a/scripts/build_ammonia_production.py b/scripts/build_ammonia_production.py index d78d627e..6f03324f 100644 --- a/scripts/build_ammonia_production.py +++ b/scripts/build_ammonia_production.py @@ -30,7 +30,7 @@ if __name__ == "__main__": ammonia.index = cc.convert(ammonia.index, to="iso2") years = [str(i) for i in range(2013, 2018)] - countries = ammonia.index.intersection(snakemake.config["countries"]) + countries = ammonia.index.intersection(snakemake.params["countries"]) ammonia = ammonia.loc[countries, years].astype(float) # convert from ktonN to ktonNH3 diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index 21d0e623..35218e2c 100644 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -210,7 +210,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_biomass_potentials", simpl="", clusters="5") - config = snakemake.config["biomass"] + config = snakemake.params["biomass"] year = config["year"] scenario = config["scenario"] diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index 6dc3b5a4..e9378792 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -116,7 +116,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_bus_regions") configure_logging(snakemake) - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] n = pypsa.Network(snakemake.input.base_network) diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py index 5d36cd5b..7128ec0d 100644 --- a/scripts/build_cop_profiles.py +++ b/scripts/build_cop_profiles.py @@ -39,7 +39,7 @@ if __name__ == "__main__": for source in ["air", "soil"]: source_T = xr.open_dataarray(snakemake.input[f"temp_{source}_{area}"]) - delta_T = snakemake.config["sector"]["heat_pump_sink_T"] - source_T + delta_T = snakemake.params["sector"]["heat_pump_sink_T"] - source_T cop = coefficient_of_performance(delta_T, source) diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 365797d2..2f61f017 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -106,9 +106,9 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_cutout", cutout="europe-2013-era5") configure_logging(snakemake) - cutout_params = snakemake.config["atlite"]["cutouts"][snakemake.wildcards.cutout] + cutout_params = snakemake.params["atlite"]["cutouts"][snakemake.wildcards.cutout] - snapshots = pd.date_range(freq="h", **snakemake.config["snapshots"]) + snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"]) time = [snapshots[0], snapshots[-1]] cutout_params["time"] = slice(*cutout_params.get("time", time)) diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py index b86b4a5f..4ef56d1d 100755 --- a/scripts/build_electricity_demand.py +++ b/scripts/build_electricity_demand.py @@ -279,16 +279,16 @@ if __name__ == "__main__": configure_logging(snakemake) - powerstatistics = snakemake.config["load"]["power_statistics"] - interpolate_limit = snakemake.config["load"]["interpolate_limit"] - countries = snakemake.config["countries"] - snapshots = pd.date_range(freq="h", **snakemake.config["snapshots"]) + powerstatistics = snakemake.params["load"]["power_statistics"] + interpolate_limit = snakemake.params["load"]["interpolate_limit"] + countries = snakemake.params["countries"] + snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"]) years = slice(snapshots[0], snapshots[-1]) - time_shift = snakemake.config["load"]["time_shift_for_large_gaps"] + time_shift = snakemake.params["load"]["time_shift_for_large_gaps"] load = load_timeseries(snakemake.input[0], years, countries, powerstatistics) - if snakemake.config["load"]["manual_adjustments"]: + if snakemake.params["load"]["manual_adjustments"]: load = manual_adjustment(load, snakemake.input[0], powerstatistics) logger.info(f"Linearly interpolate gaps of size {interpolate_limit} and less.") diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 45fc960f..6cedff97 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -373,7 +373,7 @@ def idees_per_country(ct, year, base_dir): def build_idees(countries, year): nprocesses = snakemake.threads - disable_progress = snakemake.config["run"].get("disable_progressbar", False) + disable_progress = snakemake.params["run"].get("disable_progressbar", False) func = partial(idees_per_country, year=year, base_dir=snakemake.input.idees) tqdm_kwargs = dict( @@ -735,18 +735,18 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_energy_totals") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) - config = snakemake.config["energy"] + config = snakemake.params["energy"] nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index("index") population = nuts3["pop"].groupby(nuts3.country).sum() - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] idees_countries = pd.Index(countries).intersection(eu28) data_year = config["energy_totals_year"] - report_year = snakemake.config["energy"]["eurostat_report_year"] + report_year = snakemake.params["energy"]["eurostat_report_year"] input_eurostat = snakemake.input.eurostat eurostat = build_eurostat(input_eurostat, countries, report_year, data_year) swiss = build_swiss(data_year) @@ -756,7 +756,7 @@ if __name__ == "__main__": energy.to_csv(snakemake.output.energy_name) base_year_emissions = config["base_emissions_year"] - emissions_scope = snakemake.config["energy"]["emissions"] + emissions_scope = snakemake.params["energy"]["emissions"] eea_co2 = build_eea_co2(snakemake.input.co2, base_year_emissions, emissions_scope) eurostat_co2 = build_eurostat_co2( input_eurostat, countries, report_year, base_year_emissions diff --git a/scripts/build_gas_input_locations.py b/scripts/build_gas_input_locations.py index a3b945ab..59f31f5c 100644 --- a/scripts/build_gas_input_locations.py +++ b/scripts/build_gas_input_locations.py @@ -86,7 +86,7 @@ if __name__ == "__main__": clusters="37", ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) regions = load_bus_regions( snakemake.input.regions_onshore, snakemake.input.regions_offshore diff --git a/scripts/build_gas_network.py b/scripts/build_gas_network.py index 23f58caa..a70f15ff 100644 --- a/scripts/build_gas_network.py +++ b/scripts/build_gas_network.py @@ -147,7 +147,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_gas_network") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) gas_network = load_dataset(snakemake.input.gas_network) diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py index 56ceb4b7..655df28f 100644 --- a/scripts/build_heat_demand.py +++ b/scripts/build_heat_demand.py @@ -27,7 +27,7 @@ if __name__ == "__main__": cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - time = pd.date_range(freq="h", **snakemake.config["snapshots"]) + time = pd.date_range(freq="h", **snakemake.params["snapshots"]) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) clustered_regions = ( diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 0e8cfa27..5453ac5c 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -130,10 +130,10 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_hydro_profile") configure_logging(snakemake) - config_hydro = snakemake.config["renewable"]["hydro"] + config_hydro = snakemake.params["renewable"]["hydro"] cutout = atlite.Cutout(snakemake.input.cutout) - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] country_shapes = ( gpd.read_file(snakemake.input.country_shapes) .set_index("name")["geometry"] diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index 69daf64d..30c652d8 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -73,7 +73,7 @@ def prepare_hotmaps_database(regions): df[["srid", "coordinates"]] = df.geom.str.split(";", expand=True) - if snakemake.config["industry"].get("hotmaps_locate_missing", False): + if snakemake.params["industry"].get("hotmaps_locate_missing", False): df = locate_missing_industrial_sites(df) # remove those sites without valid locations @@ -141,9 +141,9 @@ if __name__ == "__main__": clusters=48, ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] regions = gpd.read_file(snakemake.input.regions_onshore).set_index("name") diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 703997b1..4fec95cc 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -153,7 +153,7 @@ def add_non_eu28_industrial_energy_demand(countries, demand): def industrial_energy_demand(countries, year): nprocesses = snakemake.threads - disable_progress = snakemake.config["run"].get("disable_progressbar", False) + disable_progress = snakemake.params["run"].get("disable_progressbar", False) func = partial( industrial_energy_demand_per_country, year=year, jrc_dir=snakemake.input.jrc ) @@ -178,9 +178,9 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_industrial_energy_demand_per_country_today") - config = snakemake.config["industry"] + config = snakemake.params["industry"] year = config.get("reference_year", 2015) - countries = pd.Index(snakemake.config["countries"]) + countries = pd.Index(snakemake.params["countries"]) demand = industrial_energy_demand(countries.intersection(eu28), year) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 437806b3..eb1b16cb 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -217,7 +217,7 @@ def industry_production_per_country(country, year, eurostat_dir, jrc_dir): def industry_production(countries, year, eurostat_dir, jrc_dir): nprocesses = snakemake.threads - disable_progress = snakemake.config["run"].get("disable_progressbar", False) + disable_progress = snakemake.params["run"].get("disable_progressbar", False) func = partial( industry_production_per_country, @@ -277,13 +277,13 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_industrial_production_per_country") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] - year = snakemake.config["industry"]["reference_year"] + year = snakemake.params["industry"]["reference_year"] - config = snakemake.config["industry"] + config = snakemake.params["industry"] jrc_dir = snakemake.input.jrc eurostat_dir = snakemake.input.eurostat diff --git a/scripts/build_industrial_production_per_country_tomorrow.py b/scripts/build_industrial_production_per_country_tomorrow.py index 6c445608..b9ac9b16 100644 --- a/scripts/build_industrial_production_per_country_tomorrow.py +++ b/scripts/build_industrial_production_per_country_tomorrow.py @@ -15,7 +15,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_industrial_production_per_country_tomorrow") - config = snakemake.config["industry"] + config = snakemake.params["industry"] investment_year = int(snakemake.wildcards.planning_horizons) diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index 54f2cfdc..c3e2bd49 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -439,7 +439,7 @@ def chemicals_industry(): sector = "Ammonia" df[sector] = 0.0 - if snakemake.config["sector"].get("ammonia", False): + if snakemake.params["sector"].get("ammonia", False): df.loc["ammonia", sector] = config["MWh_NH3_per_tNH3"] else: df.loc["hydrogen", sector] = config["MWh_H2_per_tNH3_electrolysis"] @@ -1468,7 +1468,7 @@ if __name__ == "__main__": # TODO make config option year = 2015 - config = snakemake.config["industry"] + config = snakemake.params["industry"] df = pd.concat( [ diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index e864d925..bc35dbcf 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -23,7 +23,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_population_layouts") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) cutout = atlite.Cutout(snakemake.input.cutout) diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index 9ca67a53..bd0ee74e 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -115,7 +115,7 @@ if __name__ == "__main__": configure_logging(snakemake) n = pypsa.Network(snakemake.input.base_network) - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] ppl = ( pm.powerplants(from_url=True) @@ -134,12 +134,12 @@ if __name__ == "__main__": ppl = ppl.query('not (Country in @available_countries and Fueltype == "Bioenergy")') ppl = pd.concat([ppl, opsd]) - ppl_query = snakemake.config["electricity"]["powerplants_filter"] + ppl_query = snakemake.params["electricity"]["powerplants_filter"] if isinstance(ppl_query, str): ppl.query(ppl_query, inplace=True) # add carriers from own powerplant files: - custom_ppl_query = snakemake.config["electricity"]["custom_powerplants"] + custom_ppl_query = snakemake.params["electricity"]["custom_powerplants"] ppl = add_custom_powerplants( ppl, snakemake.input.custom_powerplants, custom_ppl_query ) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index c0288aee..55541522 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -203,8 +203,8 @@ if __name__ == "__main__": configure_logging(snakemake) nprocesses = int(snakemake.threads) - noprogress = snakemake.config["run"].get("disable_progressbar", True) - config = snakemake.config["renewable"][snakemake.wildcards.technology] + noprogress = snakemake.params["run"].get("disable_progressbar", True) + config = snakemake.params["renewable"][snakemake.wildcards.technology] resource = config["resource"] # pv panel config / wind turbine config correction_factor = config.get("correction_factor", 1.0) capacity_per_sqkm = config["capacity_per_sqkm"] diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index 9dbfc375..c52d4eb6 100644 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -305,7 +305,7 @@ def prepare_building_stock_data(): u_values.set_index(["country_code", "subsector", "bage", "type"], inplace=True) # only take in config.yaml specified countries into account - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] area_tot = area_tot.loc[countries] return u_values, country_iso_dic, countries, area_tot, area @@ -1040,7 +1040,7 @@ if __name__ == "__main__": # ******** config ********************************************************* - retro_opts = snakemake.config["sector"]["retrofitting"] + retro_opts = snakemake.params["sector"]["retrofitting"] interest_rate = retro_opts["interest_rate"] annualise_cost = retro_opts["annualise_cost"] # annualise the investment costs tax_weighting = retro_opts[ diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py index 012effe8..5c388b2e 100644 --- a/scripts/build_sequestration_potentials.py +++ b/scripts/build_sequestration_potentials.py @@ -41,7 +41,7 @@ if __name__ == "__main__": "build_sequestration_potentials", simpl="", clusters="181" ) - cf = snakemake.config["sector"]["regional_co2_sequestration_potential"] + cf = snakemake.params["sector"]["regional_co2_sequestration_potential"] gdf = gpd.read_file(snakemake.input.sequestration_potential[0]) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 50d21e12..f529517c 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -255,12 +255,12 @@ if __name__ == "__main__": configure_logging(snakemake) country_shapes = countries( - snakemake.input.naturalearth, snakemake.config["countries"] + snakemake.input.naturalearth, snakemake.params["countries"] ) country_shapes.reset_index().to_file(snakemake.output.country_shapes) offshore_shapes = eez( - country_shapes, snakemake.input.eez, snakemake.config["countries"] + country_shapes, snakemake.input.eez, snakemake.params["countries"] ) offshore_shapes.reset_index().to_file(snakemake.output.offshore_shapes) diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index f4eb1557..180007b7 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -27,9 +27,9 @@ if __name__ == "__main__": cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - config = snakemake.config["solar_thermal"] + config = snakemake.params["solar_thermal"] - time = pd.date_range(freq="h", **snakemake.config["snapshots"]) + time = pd.date_range(freq="h", **snakemake.params["snapshots"]) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) clustered_regions = ( diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index 8f6d6c6c..ee06aebb 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -27,7 +27,7 @@ if __name__ == "__main__": cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - time = pd.date_range(freq="h", **snakemake.config["snapshots"]) + time = pd.date_range(freq="h", **snakemake.params["snapshots"]) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) clustered_regions = ( diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py index 6b8bd04f..dc2b94b9 100644 --- a/scripts/build_transport_demand.py +++ b/scripts/build_transport_demand.py @@ -175,9 +175,9 @@ if __name__ == "__main__": snakemake.input.pop_weighted_energy_totals, index_col=0 ) - options = snakemake.config["sector"] + options = snakemake.params["sector"] - snapshots = pd.date_range(freq="h", **snakemake.config["snapshots"], tz="UTC") + snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"], tz="UTC") nyears = len(snapshots) / 8760 diff --git a/scripts/cluster_gas_network.py b/scripts/cluster_gas_network.py index e7554dff..ba11ce1b 100755 --- a/scripts/cluster_gas_network.py +++ b/scripts/cluster_gas_network.py @@ -110,7 +110,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("cluster_gas_network", simpl="", clusters="37") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) fn = snakemake.input.cleaned_gas_network df = pd.read_csv(fn, index_col=0) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 7572d3b3..78798804 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -468,18 +468,18 @@ if __name__ == "__main__": [ tech for tech in n.generators.carrier.unique() - if tech in snakemake.config["renewable"] + if tech in snakemake.params["renewable"] ] ) - exclude_carriers = snakemake.config["clustering"]["cluster_network"].get( + exclude_carriers = snakemake.params["clustering"]["cluster_network"].get( "exclude_carriers", [] ) aggregate_carriers = set(n.generators.carrier) - set(exclude_carriers) if snakemake.wildcards.clusters.endswith("m"): n_clusters = int(snakemake.wildcards.clusters[:-1]) conventional = set( - snakemake.config["electricity"].get("conventional_carriers", []) + snakemake.params["electricity"].get("conventional_carriers", []) ) aggregate_carriers = conventional.intersection(aggregate_carriers) elif snakemake.wildcards.clusters == "all": @@ -495,13 +495,13 @@ if __name__ == "__main__": n, busmap, linemap, linemap, pd.Series(dtype="O") ) else: - line_length_factor = snakemake.config["lines"]["length_factor"] + line_length_factor = snakemake.params["lines"]["length_factor"] Nyears = n.snapshot_weightings.objective.sum() / 8760 hvac_overhead_cost = load_costs( snakemake.input.tech_costs, - snakemake.config["costs"], - snakemake.config["electricity"], + snakemake.params["costs"], + snakemake.params["electricity"], Nyears, ).at["HVAC overhead", "capital_cost"] @@ -512,7 +512,7 @@ if __name__ == "__main__": ).all() or x.isnull().all(), "The `potential` configuration option must agree for all renewable carriers, for now!" return v - aggregation_strategies = snakemake.config["clustering"].get( + aggregation_strategies = snakemake.params["clustering"].get( "aggregation_strategies", {} ) # translate str entries of aggregation_strategies to pd.Series functions: @@ -521,7 +521,7 @@ if __name__ == "__main__": for p in aggregation_strategies.keys() } - custom_busmap = snakemake.config["enable"].get("custom_busmap", False) + custom_busmap = snakemake.params["enable"].get("custom_busmap", False) if custom_busmap: custom_busmap = pd.read_csv( snakemake.input.custom_busmap, index_col=0, squeeze=True @@ -539,7 +539,7 @@ if __name__ == "__main__": aggregate_carriers, line_length_factor, aggregation_strategies, - snakemake.config["solving"]["solver"]["name"], + snakemake.params["solving"]["solver"]["name"], cluster_config.get("algorithm", "hac"), cluster_config.get("feature", "solar+onwind-time"), hvac_overhead_cost, diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 3d743942..56074525 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -198,7 +198,7 @@ def calculate_costs(n, label, costs): def calculate_cumulative_cost(): - planning_horizons = snakemake.config["scenario"]["planning_horizons"] + planning_horizons = snakemake.params["scenario"]["planning_horizons"] cumulative_cost = pd.DataFrame( index=df["costs"].sum().index, @@ -682,25 +682,25 @@ if __name__ == "__main__": snakemake = mock_snakemake("make_summary") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) networks_dict = { (cluster, ll, opt + sector_opt, planning_horizon): "results/" + snakemake.params.RDIR + f"/postnetworks/elec_s{simpl}_{cluster}_l{ll}_{opt}_{sector_opt}_{planning_horizon}.nc" - for simpl in snakemake.config["scenario"]["simpl"] - for cluster in snakemake.config["scenario"]["clusters"] - for opt in snakemake.config["scenario"]["opts"] - for sector_opt in snakemake.config["scenario"]["sector_opts"] - for ll in snakemake.config["scenario"]["ll"] - for planning_horizon in snakemake.config["scenario"]["planning_horizons"] + for simpl in snakemake.params["scenario"]["simpl"] + for cluster in snakemake.params["scenario"]["clusters"] + for opt in snakemake.params["scenario"]["opts"] + for sector_opt in snakemake.params["scenario"]["sector_opts"] + for ll in snakemake.params["scenario"]["ll"] + for planning_horizon in snakemake.params["scenario"]["planning_horizons"] } - Nyears = len(pd.date_range(freq="h", **snakemake.config["snapshots"])) / 8760 + Nyears = len(pd.date_range(freq="h", **snakemake.params["snapshots"])) / 8760 costs_db = prepare_costs( snakemake.input.costs, - snakemake.config["costs"], + snakemake.params["costs"], Nyears, ) @@ -710,7 +710,7 @@ if __name__ == "__main__": to_csv(df) - if snakemake.config["foresight"] == "myopic": + if snakemake.params["foresight"] == "myopic": cumulative_cost = calculate_cumulative_cost() cumulative_cost.to_csv( "results/" + snakemake.params.RDIR + "/csvs/cumulative_cost.csv" diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 0a22b2e5..8aac9db1 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -70,7 +70,7 @@ def plot_map( transmission=False, with_legend=True, ): - tech_colors = snakemake.config["plotting"]["tech_colors"] + tech_colors = snakemake.params["plotting"]["tech_colors"] n = network.copy() assign_location(n) @@ -116,7 +116,7 @@ def plot_map( costs = costs.stack() # .sort_index() # hack because impossible to drop buses... - eu_location = snakemake.config["plotting"].get( + eu_location = snakemake.params["plotting"].get( "eu_node_location", dict(x=-5.5, y=46) ) n.buses.loc["EU gas", "x"] = eu_location["x"] @@ -315,7 +315,7 @@ def plot_h2_map(network, regions): h2_new = n.links[n.links.carrier == "H2 pipeline"] h2_retro = n.links[n.links.carrier == "H2 pipeline retrofitted"] - if snakemake.config["foresight"] == "myopic": + if snakemake.params["foresight"] == "myopic": # sum capacitiy for pipelines from different investment periods h2_new = group_pipes(h2_new) @@ -558,7 +558,7 @@ def plot_ch4_map(network): link_widths_used = max_usage / linewidth_factor link_widths_used[max_usage < line_lower_threshold] = 0.0 - tech_colors = snakemake.config["plotting"]["tech_colors"] + tech_colors = snakemake.params["plotting"]["tech_colors"] pipe_colors = { "gas pipeline": "#f08080", @@ -700,7 +700,7 @@ def plot_map_without(network): # hack because impossible to drop buses... if "EU gas" in n.buses.index: - eu_location = snakemake.config["plotting"].get( + eu_location = snakemake.params["plotting"].get( "eu_node_location", dict(x=-5.5, y=46) ) n.buses.loc["EU gas", "x"] = eu_location["x"] @@ -876,7 +876,7 @@ def plot_series(network, carrier="AC", name="test"): stacked=True, linewidth=0.0, color=[ - snakemake.config["plotting"]["tech_colors"][i.replace(suffix, "")] + snakemake.params["plotting"]["tech_colors"][i.replace(suffix, "")] for i in new_columns ], ) @@ -930,14 +930,14 @@ if __name__ == "__main__": planning_horizons="2030", ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) regions = gpd.read_file(snakemake.input.regions).set_index("name") - map_opts = snakemake.config["plotting"]["map"] + map_opts = snakemake.params["plotting"]["map"] if map_opts["boundaries"] is None: map_opts["boundaries"] = regions.total_bounds[[0, 2, 1, 3]] + [-1, 1, -1, 1] diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index cfa8e361..e18ff61b 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -142,10 +142,10 @@ def plot_costs(): df = df.groupby(df.index.map(rename_techs)).sum() - to_drop = df.index[df.max(axis=1) < snakemake.config["plotting"]["costs_threshold"]] + to_drop = df.index[df.max(axis=1) < snakemake.params["plotting"]["costs_threshold"]] logger.info( - f"Dropping technology with costs below {snakemake.config['plotting']['costs_threshold']} EUR billion per year" + f"Dropping technology with costs below {snakemake.params['plotting']['costs_threshold']} EUR billion per year" ) logger.debug(df.loc[to_drop]) @@ -165,7 +165,7 @@ def plot_costs(): kind="bar", ax=ax, stacked=True, - color=[snakemake.config["plotting"]["tech_colors"][i] for i in new_index], + color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index], ) handles, labels = ax.get_legend_handles_labels() @@ -173,7 +173,7 @@ def plot_costs(): handles.reverse() labels.reverse() - ax.set_ylim([0, snakemake.config["plotting"]["costs_max"]]) + ax.set_ylim([0, snakemake.params["plotting"]["costs_max"]]) ax.set_ylabel("System Cost [EUR billion per year]") @@ -201,11 +201,11 @@ def plot_energy(): df = df.groupby(df.index.map(rename_techs)).sum() to_drop = df.index[ - df.abs().max(axis=1) < snakemake.config["plotting"]["energy_threshold"] + df.abs().max(axis=1) < snakemake.params["plotting"]["energy_threshold"] ] logger.info( - f"Dropping all technology with energy consumption or production below {snakemake.config['plotting']['energy_threshold']} TWh/a" + f"Dropping all technology with energy consumption or production below {snakemake.params['plotting']['energy_threshold']} TWh/a" ) logger.debug(df.loc[to_drop]) @@ -227,7 +227,7 @@ def plot_energy(): kind="bar", ax=ax, stacked=True, - color=[snakemake.config["plotting"]["tech_colors"][i] for i in new_index], + color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index], ) handles, labels = ax.get_legend_handles_labels() @@ -237,8 +237,8 @@ def plot_energy(): ax.set_ylim( [ - snakemake.config["plotting"]["energy_min"], - snakemake.config["plotting"]["energy_max"], + snakemake.params["plotting"]["energy_min"], + snakemake.params["plotting"]["energy_max"], ] ) @@ -287,7 +287,7 @@ def plot_balances(): df = df.groupby(df.index.map(rename_techs)).sum() to_drop = df.index[ - df.abs().max(axis=1) < snakemake.config["plotting"]["energy_threshold"] / 10 + df.abs().max(axis=1) < snakemake.params["plotting"]["energy_threshold"] / 10 ] if v[0] in co2_carriers: @@ -296,7 +296,7 @@ def plot_balances(): units = "TWh/a" logger.debug( - f"Dropping technology energy balance smaller than {snakemake.config['plotting']['energy_threshold']/10} {units}" + f"Dropping technology energy balance smaller than {snakemake.params['plotting']['energy_threshold']/10} {units}" ) logger.debug(df.loc[to_drop]) @@ -317,7 +317,7 @@ def plot_balances(): kind="bar", ax=ax, stacked=True, - color=[snakemake.config["plotting"]["tech_colors"][i] for i in new_index], + color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index], ) handles, labels = ax.get_legend_handles_labels() @@ -455,10 +455,10 @@ def plot_carbon_budget_distribution(input_eurostat): ax1 = plt.subplot(gs1[0, 0]) ax1.set_ylabel("CO$_2$ emissions (Gt per year)", fontsize=22) ax1.set_ylim([0, 5]) - ax1.set_xlim([1990, snakemake.config["scenario"]["planning_horizons"][-1] + 1]) + ax1.set_xlim([1990, snakemake.params["scenario"]["planning_horizons"][-1] + 1]) path_cb = "results/" + snakemake.params.RDIR + "/csvs/" - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] e_1990 = co2_emissions_year(countries, input_eurostat, opts, year=1990) CO2_CAP = pd.read_csv(path_cb + "carbon_budget_distribution.csv", index_col=0) @@ -545,7 +545,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("plot_summary") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) n_header = 4 @@ -555,7 +555,7 @@ if __name__ == "__main__": plot_balances() - for sector_opts in snakemake.config["scenario"]["sector_opts"]: + for sector_opts in snakemake.params["scenario"]["sector_opts"]: opts = sector_opts.split("-") for o in opts: if "cb" in o: diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 14a003ae..46e4e74a 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -253,12 +253,12 @@ if __name__ == "__main__": Nyears = n.snapshot_weightings.objective.sum() / 8760.0 costs = load_costs( snakemake.input.tech_costs, - snakemake.config["costs"], - snakemake.config["electricity"], + snakemake.params["costs"], + snakemake.params["electricity"], Nyears, ) - set_line_s_max_pu(n, snakemake.config["lines"]["s_max_pu"]) + set_line_s_max_pu(n, snakemake.params["lines"]["s_max_pu"]) for o in opts: m = re.match(r"^\d+h$", o, re.IGNORECASE) @@ -269,7 +269,7 @@ if __name__ == "__main__": for o in opts: m = re.match(r"^\d+seg$", o, re.IGNORECASE) if m is not None: - solver_name = snakemake.config["solving"]["solver"]["name"] + solver_name = snakemake.params["solving"]["solver"]["name"] n = apply_time_segmentation(n, m.group(0)[:-3], solver_name) break @@ -277,11 +277,11 @@ if __name__ == "__main__": if "Co2L" in o: m = re.findall("[0-9]*\.?[0-9]+$", o) if len(m) > 0: - co2limit = float(m[0]) * snakemake.config["electricity"]["co2base"] + co2limit = float(m[0]) * snakemake.params["electricity"]["co2base"] add_co2limit(n, co2limit, Nyears) logger.info("Setting CO2 limit according to wildcard value.") else: - add_co2limit(n, snakemake.config["electricity"]["co2limit"], Nyears) + add_co2limit(n, snakemake.params["electricity"]["co2limit"], Nyears) logger.info("Setting CO2 limit according to config value.") break @@ -293,7 +293,7 @@ if __name__ == "__main__": add_gaslimit(n, limit, Nyears) logger.info("Setting gas usage limit according to wildcard value.") else: - add_gaslimit(n, snakemake.config["electricity"].get("gaslimit"), Nyears) + add_gaslimit(n, snakemake.params["electricity"].get("gaslimit"), Nyears) logger.info("Setting gas usage limit according to config value.") break @@ -322,7 +322,7 @@ if __name__ == "__main__": add_emission_prices(n, dict(co2=float(m[0]))) else: logger.info("Setting emission prices according to config value.") - add_emission_prices(n, snakemake.config["costs"]["emission_prices"]) + add_emission_prices(n, snakemake.params["costs"]["emission_prices"]) break ll_type, factor = snakemake.wildcards.ll[0], snakemake.wildcards.ll[1:] @@ -330,8 +330,8 @@ if __name__ == "__main__": set_line_nom_max( n, - s_nom_max_set=snakemake.config["lines"].get("s_nom_max,", np.inf), - p_nom_max_set=snakemake.config["links"].get("p_nom_max,", np.inf), + s_nom_max_set=snakemake.params["lines"].get("s_nom_max,", np.inf), + p_nom_max_set=snakemake.params["links"].get("p_nom_max,", np.inf), ) if "ATK" in opts: diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 1cb7146e..c69f7290 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -200,12 +200,12 @@ def co2_emissions_year( """ Calculate CO2 emissions in one specific year (e.g. 1990 or 2018). """ - emissions_scope = snakemake.config["energy"]["emissions"] + emissions_scope = snakemake.params["energy"]["emissions"] eea_co2 = build_eea_co2(snakemake.input.co2, year, emissions_scope) # TODO: read Eurostat data from year > 2014 # this only affects the estimation of CO2 emissions for BA, RS, AL, ME, MK - report_year = snakemake.config["energy"]["eurostat_report_year"] + report_year = snakemake.params["energy"]["eurostat_report_year"] if year > 2014: eurostat_co2 = build_eurostat_co2( input_eurostat, countries, report_year, year=2014 @@ -241,7 +241,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): carbon_budget = float(o[o.find("cb") + 2 : o.find("ex")]) r = float(o[o.find("ex") + 2 :]) - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] e_1990 = co2_emissions_year( countries, input_eurostat, opts, emissions_scope, report_year, year=1990 @@ -252,7 +252,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): countries, input_eurostat, opts, emissions_scope, report_year, year=2018 ) - planning_horizons = snakemake.config["scenario"]["planning_horizons"] + planning_horizons = snakemake.params["scenario"]["planning_horizons"] t_0 = planning_horizons[0] if "be" in o: @@ -391,7 +391,7 @@ def update_wind_solar_costs(n, costs): with xr.open_dataset(profile) as ds: underwater_fraction = ds["underwater_fraction"].to_pandas() connection_cost = ( - snakemake.config["lines"]["length_factor"] + snakemake.params["lines"]["length_factor"] * ds["average_distance"].to_pandas() * ( underwater_fraction @@ -483,8 +483,8 @@ def remove_elec_base_techs(n): batteries and H2) from base electricity-only network, since they're added here differently using links. """ - for c in n.iterate_components(snakemake.config["pypsa_eur"]): - to_keep = snakemake.config["pypsa_eur"][c.name] + for c in n.iterate_components(snakemake.params["pypsa_eur"]): + to_keep = snakemake.params["pypsa_eur"][c.name] to_remove = pd.Index(c.df.carrier.unique()).symmetric_difference(to_keep) if to_remove.empty: continue @@ -674,7 +674,7 @@ def add_dac(n, costs): def add_co2limit(n, nyears=1.0, limit=0.0): logger.info(f"Adding CO2 budget limit as per unit of 1990 levels of {limit}") - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] sectors = emission_sectors_from_opts(opts) @@ -787,7 +787,7 @@ def add_ammonia(n, costs): nodes = pop_layout.index - cf_industry = snakemake.config["industry"] + cf_industry = snakemake.params["industry"] n.add("Carrier", "NH3") @@ -1102,7 +1102,7 @@ def add_storage_and_grids(n, costs): lifetime=costs.at["OCGT", "lifetime"], ) - cavern_types = snakemake.config["sector"]["hydrogen_underground_storage_locations"] + cavern_types = snakemake.params["sector"]["hydrogen_underground_storage_locations"] h2_caverns = pd.read_csv(snakemake.input.h2_cavern, index_col=0) if not h2_caverns.empty and options["hydrogen_underground_storage"]: @@ -3266,11 +3266,11 @@ if __name__ == "__main__": planning_horizons="2030", ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) - options = snakemake.config["sector"] + options = snakemake.params["sector"] opts = snakemake.wildcards.sector_opts.split("-") @@ -3285,7 +3285,7 @@ if __name__ == "__main__": costs = prepare_costs( snakemake.input.costs, - snakemake.config["costs"], + snakemake.params["costs"], nyears, ) @@ -3297,10 +3297,10 @@ if __name__ == "__main__": spatial = define_spatial(pop_layout.index, options) - if snakemake.config["foresight"] == "myopic": + if snakemake.params["foresight"] == "myopic": add_lifetime_wind_solar(n, costs) - conventional = snakemake.config["existing_capacities"]["conventional_carriers"] + conventional = snakemake.params["existing_capacities"]["conventional_carriers"] for carrier in conventional: add_carrier_buses(n, carrier) @@ -3365,19 +3365,19 @@ if __name__ == "__main__": if options["allam_cycle"]: add_allam(n, costs) - solver_name = snakemake.config["solving"]["solver"]["name"] + solver_name = snakemake.params["solving"]["solver"]["name"] n = set_temporal_aggregation(n, opts, solver_name) limit_type = "config" - limit = get(snakemake.config["co2_budget"], investment_year) + limit = get(snakemake.params["co2_budget"], investment_year) for o in opts: if "cb" not in o: continue limit_type = "carbon budget" fn = "results/" + snakemake.params.RDIR + "/csvs/carbon_budget_distribution.csv" if not os.path.exists(fn): - emissions_scope = snakemake.config["energy"]["emissions"] - report_year = snakemake.config["energy"]["eurostat_report_year"] + emissions_scope = snakemake.params["energy"]["emissions"] + report_year = snakemake.params["energy"]["eurostat_report_year"] build_carbon_budget( o, snakemake.input.eurostat, fn, emissions_scope, report_year ) @@ -3412,8 +3412,8 @@ if __name__ == "__main__": if options["electricity_grid_connection"]: add_electricity_grid_connection(n, costs) - first_year_myopic = (snakemake.config["foresight"] == "myopic") and ( - snakemake.config["scenario"]["planning_horizons"][0] == investment_year + first_year_myopic = (snakemake.params["foresight"] == "myopic") and ( + snakemake.params["scenario"]["planning_horizons"][0] == investment_year ) if options.get("cluster_heat_buses", False) and not first_year_myopic: diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index 0c6a7feb..0271166c 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -53,7 +53,7 @@ if __name__ == "__main__": snakemake ) # TODO Make logging compatible with progressbar (see PR #102) - if snakemake.config["tutorial"]: + if snakemake.params["tutorial"]: url = "https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz" else: url = "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz" @@ -63,7 +63,7 @@ if __name__ == "__main__": to_fn = Path(f"{rootpath}/data") logger.info(f"Downloading databundle from '{url}'.") - disable_progress = snakemake.config["run"].get("disable_progressbar", False) + disable_progress = snakemake.params["run"].get("disable_progressbar", False) progress_retrieve(url, tarball_fn, disable=disable_progress) logger.info("Extracting databundle.") diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 2be8c36a..bb7f615c 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -510,7 +510,7 @@ if __name__ == "__main__": n = pypsa.Network(snakemake.input.network) - aggregation_strategies = snakemake.config["clustering"].get( + aggregation_strategies = snakemake.params["clustering"].get( "aggregation_strategies", {} ) # translate str entries of aggregation_strategies to pd.Series functions: @@ -525,8 +525,8 @@ if __name__ == "__main__": technology_costs = load_costs( snakemake.input.tech_costs, - snakemake.config["costs"], - snakemake.config["electricity"], + snakemake.params["costs"], + snakemake.params["electricity"], Nyears, ) @@ -536,7 +536,7 @@ if __name__ == "__main__": busmaps = [trafo_map, simplify_links_map] - cluster_config = snakemake.config["clustering"]["simplify_network"] + cluster_config = snakemake.params["clustering"]["simplify_network"] if cluster_config.get("remove_stubs", True): n, stub_map = remove_stubs( n, diff --git a/scripts/solve_network.py b/scripts/solve_network.py index ff1c0ccf..c7041e85 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -663,7 +663,7 @@ if __name__ == "__main__": if "sector_opts" in snakemake.wildcards.keys(): opts += "-" + snakemake.wildcards.sector_opts opts = [o for o in opts.split("-") if o != ""] - solve_opts = snakemake.config["solving"]["options"] + solve_opts = snakemake.params["solving"]["options"] np.random.seed(solve_opts.get("seed", 123)) diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 25fe0753..c1b2be6f 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -42,7 +42,7 @@ if __name__ == "__main__": opts = (snakemake.wildcards.opts + "-" + snakemake.wildcards.sector_opts).split("-") opts = [o for o in opts if o != ""] - solve_opts = snakemake.config["solving"]["options"] + solve_opts = snakemake.params["solving"]["options"] np.random.seed(solve_opts.get("seed", 123)) From 61893c3c9b9e9f738d795f31ab03be07f6558ebd Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Mon, 15 May 2023 11:26:36 +0200 Subject: [PATCH 09/41] fixing build_electricity.smk and retrieve.smk --- rules/build_electricity.smk | 1 + rules/retrieve.smk | 1 + 2 files changed, 2 insertions(+) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index bb000f2b..56098ddd 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -201,6 +201,7 @@ rule build_ship_raster: rule build_renewable_profiles: params: + run=config["run"], renewable=config["renewable"], input: base_network=RESOURCES + "networks/base.nc", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 4bfbd6c6..bc8756d5 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -20,6 +20,7 @@ if config["enable"].get("retrieve_databundle", True): rule retrieve_databundle: params: + run=config["run"], tutorial=config["tutorial"], output: expand("data/bundle/{file}", file=datafiles), From 748cfaec65aeaa1b4a671ac854aaa46cfd0941e7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 May 2023 22:01:44 +0000 Subject: [PATCH 10/41] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/macisamuele/language-formatters-pre-commit-hooks: v2.8.0 → v2.9.0](https://github.com/macisamuele/language-formatters-pre-commit-hooks/compare/v2.8.0...v2.9.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 88685f3b..48915f4c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -67,7 +67,7 @@ repos: # Do YAML formatting (before the linter checks it for misses) - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks - rev: v2.8.0 + rev: v2.9.0 hooks: - id: pretty-format-yaml args: [--autofix, --indent, "2", --preserve-quotes] From 875fcdd0e2ee1ef85e9d769446f1ebf7e836b4e0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 16 May 2023 13:34:23 +0200 Subject: [PATCH 11/41] address geopandas future warning --- scripts/build_shapes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 50d21e12..54a7774e 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -234,6 +234,7 @@ def nuts3(country_shapes, nuts3, nuts3pop, nuts3gdp, ch_cantons, ch_popgdp): manual = gpd.GeoDataFrame( [["BA1", "BA", 3871.0], ["RS1", "RS", 7210.0], ["AL1", "AL", 2893.0]], columns=["NUTS_ID", "country", "pop"], + geometry=gpd.GeoSeries() ) manual["geometry"] = manual["country"].map(country_shapes) manual = manual.dropna() From 7d4dacf8bdeb24513888719065069102152b9be4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 May 2023 11:34:58 +0000 Subject: [PATCH 12/41] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_shapes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 54a7774e..5561cfb2 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -234,7 +234,7 @@ def nuts3(country_shapes, nuts3, nuts3pop, nuts3gdp, ch_cantons, ch_popgdp): manual = gpd.GeoDataFrame( [["BA1", "BA", 3871.0], ["RS1", "RS", 7210.0], ["AL1", "AL", 2893.0]], columns=["NUTS_ID", "country", "pop"], - geometry=gpd.GeoSeries() + geometry=gpd.GeoSeries(), ) manual["geometry"] = manual["country"].map(country_shapes) manual = manual.dropna() From 7fad9e263569db7eaf392c56e47e01c552343342 Mon Sep 17 00:00:00 2001 From: Lissy Langer <54096244+lilanger@users.noreply.github.com> Date: Tue, 16 May 2023 14:57:45 +0200 Subject: [PATCH 13/41] Update supply_demand.rst Heat pump COP equations signs are wrong: https://pubs.rsc.org/en/content/articlelanding/2012/EE/c2ee22653g --- doc/supply_demand.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/supply_demand.rst b/doc/supply_demand.rst index 16242405..b043268b 100644 --- a/doc/supply_demand.rst +++ b/doc/supply_demand.rst @@ -133,12 +133,12 @@ The coefficient of performance (COP) of air- and ground-sourced heat pumps depen For the sink water temperature Tsink we assume 55 °C [`Config `_ file]. For the time- and location-dependent source temperatures Tsource, we rely on the `ERA5 `_ reanalysis weather data. The temperature differences are converted into COP time series using results from a regression analysis performed in the study by `Stafell et al. `_. For air-sourced heat pumps (ASHP), we use the function: .. math:: - COP (\Delta T) = 6.81 + 0.121\Delta T + 0.000630\Delta T^2 + COP (\Delta T) = 6.81 - 0.121\Delta T + 0.000630\Delta T^2 for ground-sourced heat pumps (GSHP), we use the function: .. math:: - COP(\Delta T) = 8.77 + 0.150\Delta T + 0.000734\Delta T^2 + COP(\Delta T) = 8.77 - 0.150\Delta T + 0.000734\Delta T^2 **Resistive heaters** From 8d6d6610712e1f5a2ce3bfab13a1a23afec21029 Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Tue, 16 May 2023 20:44:22 +0100 Subject: [PATCH 14/41] Update support.rst --- doc/support.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/support.rst b/doc/support.rst index 36d1a2dd..1d512d59 100644 --- a/doc/support.rst +++ b/doc/support.rst @@ -9,6 +9,6 @@ Support * In case of code-related **questions**, please post on `stack overflow `_. * For non-programming related and more general questions please refer to the `mailing list `_. -* To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.gg/JTdvaEBb). +* To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the `discord server `_. * For **bugs and feature requests**, please use the `issue tracker `_. * We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github `_. For further information on how to contribute, please refer to :ref:`contributing`. From 8208ac033baff982fbeedb11ef019d4bd581576c Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Wed, 17 May 2023 18:43:30 +0200 Subject: [PATCH 15/41] convert param back to config for "logging" and "run", made "enable" more specific --- rules/build_electricity.smk | 3 +-- rules/build_sector.smk | 15 --------------- rules/postprocess.smk | 3 --- rules/retrieve.smk | 1 - rules/solve_myopic.smk | 2 -- scripts/add_brownfield.py | 2 +- scripts/add_existing_baseyear.py | 2 +- scripts/build_energy_totals.py | 4 ++-- scripts/build_gas_input_locations.py | 2 +- scripts/build_gas_network.py | 2 +- scripts/build_industrial_distribution_key.py | 2 +- ..._industrial_energy_demand_per_country_today.py | 2 +- .../build_industrial_production_per_country.py | 4 ++-- scripts/build_population_layouts.py | 2 +- scripts/build_renewable_profiles.py | 2 +- scripts/cluster_gas_network.py | 2 +- scripts/make_summary.py | 2 +- scripts/plot_network.py | 2 +- scripts/plot_summary.py | 2 +- scripts/prepare_sector_network.py | 2 +- scripts/retrieve_databundle.py | 2 +- 21 files changed, 19 insertions(+), 41 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 56098ddd..9414c25b 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -201,7 +201,6 @@ rule build_ship_raster: rule build_renewable_profiles: params: - run=config["run"], renewable=config["renewable"], input: base_network=RESOURCES + "networks/base.nc", @@ -350,7 +349,7 @@ rule cluster_network: lines=config["lines"], renewable=config["renewable"], clustering=config["clustering"], - enable=config["enable"], + enable=config["enable"].get("custom_busmap", False), input: network=RESOURCES + "networks/elec_s{simpl}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson", diff --git a/rules/build_sector.smk b/rules/build_sector.smk index d375b7b9..9faae4b9 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -4,8 +4,6 @@ rule build_population_layouts: - params: - logging=config["logging"], input: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", urban_percent="data/urban_percent.csv", @@ -72,8 +70,6 @@ rule build_simplified_population_layouts: if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: rule build_gas_network: - params: - logging=config["logging"], input: gas_network="data/gas_network/scigrid-gas/data/IGGIELGN_PipeSegments.geojson", output: @@ -88,8 +84,6 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: "../scripts/build_gas_network.py" rule build_gas_input_locations: - params: - logging=config["logging"], input: lng=HTTP.remote( "https://globalenergymonitor.org/wp-content/uploads/2022/09/Europe-Gas-Tracker-August-2022.xlsx", @@ -116,8 +110,6 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: "../scripts/build_gas_input_locations.py" rule cluster_gas_network: - params: - logging=config["logging"], input: cleaned_gas_network=RESOURCES + "gas_network.csv", regions_onshore=RESOURCES @@ -246,10 +238,8 @@ rule build_solar_thermal_profiles: rule build_energy_totals: params: - run=config["run"], countries=config["countries"], energy=config["energy"], - logging=config["logging"], input: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", co2="data/eea/UNFCCC_v23.csv", @@ -437,10 +427,8 @@ rule build_industry_sector_ratios: rule build_industrial_production_per_country: params: - run=config["run"], industry=config["industry"], countries=config["countries"], - logging=config["logging"], input: ammonia_production=RESOURCES + "ammonia_production.csv", jrc="data/jrc-idees-2015", @@ -491,7 +479,6 @@ rule build_industrial_distribution_key: params: industry=config["industry"], countries=config["countries"], - logging=config["logging"], input: regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", @@ -567,7 +554,6 @@ rule build_industrial_energy_demand_per_node: rule build_industrial_energy_demand_per_country_today: params: - run=config["run"], countries=config["countries"], industry=config["industry"], input: @@ -723,7 +709,6 @@ rule prepare_sector_network: existing_capacities=config["existing_capacities"], foresight=config["foresight"], costs=config["costs"], - logging=config["logging"], sector=config["sector"], industry=config["industry"], pypsa_eur=config["pypsa_eur"], diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 1dfdd098..9eb04ef5 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -10,7 +10,6 @@ localrules: rule plot_network: params: - logging=config["logging"], foresight=config["foresight"], plotting=config["plotting"], input: @@ -74,7 +73,6 @@ rule make_summary: foresight=config["foresight"], costs=config["costs"], snapshots=config["snapshots"], - logging=config["logging"], scenario=config["scenario"], RDIR=RDIR, input: @@ -123,7 +121,6 @@ rule make_summary: rule plot_summary: params: - logging=config["logging"], countries=config["countries"], scenario=config["scenario"], plotting=config["plotting"], diff --git a/rules/retrieve.smk b/rules/retrieve.smk index bc8756d5..4bfbd6c6 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -20,7 +20,6 @@ if config["enable"].get("retrieve_databundle", True): rule retrieve_databundle: params: - run=config["run"], tutorial=config["tutorial"], output: expand("data/bundle/{file}", file=datafiles), diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 2ecba999..49b93a80 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -7,7 +7,6 @@ rule add_existing_baseyear: params: scenario=config["scenario"], sector=config["sector"], - logging=config["logging"], existing_capacities=config["existing_capacities"], costs=config["costs"], input: @@ -49,7 +48,6 @@ rule add_existing_baseyear: rule add_brownfield: params: - logging=config["logging"], sector=config["sector"], existing_capacities=config["existing_capacities"], input: diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 1d4e3a80..9330953b 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -139,7 +139,7 @@ if __name__ == "__main__": planning_horizons=2030, ) - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index a24b078d..a4d3748b 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -608,7 +608,7 @@ if __name__ == "__main__": planning_horizons=2020, ) - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 6cedff97..5099d140 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -373,7 +373,7 @@ def idees_per_country(ct, year, base_dir): def build_idees(countries, year): nprocesses = snakemake.threads - disable_progress = snakemake.params["run"].get("disable_progressbar", False) + disable_progress = snakemake.config["run"].get("disable_progressbar", False) func = partial(idees_per_country, year=year, base_dir=snakemake.input.idees) tqdm_kwargs = dict( @@ -735,7 +735,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_energy_totals") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) config = snakemake.params["energy"] diff --git a/scripts/build_gas_input_locations.py b/scripts/build_gas_input_locations.py index 59f31f5c..a3b945ab 100644 --- a/scripts/build_gas_input_locations.py +++ b/scripts/build_gas_input_locations.py @@ -86,7 +86,7 @@ if __name__ == "__main__": clusters="37", ) - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) regions = load_bus_regions( snakemake.input.regions_onshore, snakemake.input.regions_offshore diff --git a/scripts/build_gas_network.py b/scripts/build_gas_network.py index a70f15ff..23f58caa 100644 --- a/scripts/build_gas_network.py +++ b/scripts/build_gas_network.py @@ -147,7 +147,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_gas_network") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) gas_network = load_dataset(snakemake.input.gas_network) diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index 30c652d8..9a513673 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -141,7 +141,7 @@ if __name__ == "__main__": clusters=48, ) - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) countries = snakemake.params["countries"] diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 4fec95cc..c28351c1 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -153,7 +153,7 @@ def add_non_eu28_industrial_energy_demand(countries, demand): def industrial_energy_demand(countries, year): nprocesses = snakemake.threads - disable_progress = snakemake.params["run"].get("disable_progressbar", False) + disable_progress = snakemake.config["run"].get("disable_progressbar", False) func = partial( industrial_energy_demand_per_country, year=year, jrc_dir=snakemake.input.jrc ) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index eb1b16cb..62073ea1 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -217,7 +217,7 @@ def industry_production_per_country(country, year, eurostat_dir, jrc_dir): def industry_production(countries, year, eurostat_dir, jrc_dir): nprocesses = snakemake.threads - disable_progress = snakemake.params["run"].get("disable_progressbar", False) + disable_progress = snakemake.config["run"].get("disable_progressbar", False) func = partial( industry_production_per_country, @@ -277,7 +277,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_industrial_production_per_country") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) countries = snakemake.params["countries"] diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index bc35dbcf..e864d925 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -23,7 +23,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_population_layouts") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) cutout = atlite.Cutout(snakemake.input.cutout) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 55541522..8fba74e6 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -203,7 +203,7 @@ if __name__ == "__main__": configure_logging(snakemake) nprocesses = int(snakemake.threads) - noprogress = snakemake.params["run"].get("disable_progressbar", True) + noprogress = snakemake.config["run"].get("disable_progressbar", True) config = snakemake.params["renewable"][snakemake.wildcards.technology] resource = config["resource"] # pv panel config / wind turbine config correction_factor = config.get("correction_factor", 1.0) diff --git a/scripts/cluster_gas_network.py b/scripts/cluster_gas_network.py index ba11ce1b..e7554dff 100755 --- a/scripts/cluster_gas_network.py +++ b/scripts/cluster_gas_network.py @@ -110,7 +110,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("cluster_gas_network", simpl="", clusters="37") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) fn = snakemake.input.cleaned_gas_network df = pd.read_csv(fn, index_col=0) diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 56074525..da0712d7 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -682,7 +682,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("make_summary") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) networks_dict = { (cluster, ll, opt + sector_opt, planning_horizon): "results/" diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 8aac9db1..399f46e8 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -930,7 +930,7 @@ if __name__ == "__main__": planning_horizons="2030", ) - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index e18ff61b..4aa37de5 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -545,7 +545,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("plot_summary") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) n_header = 4 diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index c69f7290..86a9bc82 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3266,7 +3266,7 @@ if __name__ == "__main__": planning_horizons="2030", ) - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index 0271166c..de42587d 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -63,7 +63,7 @@ if __name__ == "__main__": to_fn = Path(f"{rootpath}/data") logger.info(f"Downloading databundle from '{url}'.") - disable_progress = snakemake.params["run"].get("disable_progressbar", False) + disable_progress = snakemake.config["run"].get("disable_progressbar", False) progress_retrieve(url, tarball_fn, disable=disable_progress) logger.info("Extracting databundle.") From 8af1fe5649feb6df991db0d5b991c66fc9e8c25e Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Wed, 17 May 2023 19:25:45 +0200 Subject: [PATCH 16/41] replace the word "config" to "params" in functions --- rules/build_electricity.smk | 1 + scripts/add_electricity.py | 47 +++++++++-------- scripts/build_biomass_potentials.py | 8 +-- scripts/build_energy_totals.py | 6 +-- scripts/build_hydro_profile.py | 6 +-- ...ustrial_energy_demand_per_country_today.py | 10 ++-- ...build_industrial_production_per_country.py | 8 +-- ...ustrial_production_per_country_tomorrow.py | 14 +++--- scripts/build_industry_sector_ratios.py | 50 +++++++++---------- scripts/build_renewable_profiles.py | 48 +++++++++--------- scripts/prepare_sector_network.py | 4 +- 11 files changed, 101 insertions(+), 101 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 9414c25b..88e7c548 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -277,6 +277,7 @@ rule add_electricity: countries=config["countries"], renewable=config["renewable"], electricity=config["electricity"], + conventional=config.get("conventional", {}) costs=config["costs"], input: **{ diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index c3e847ce..a3e033e7 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -135,7 +135,7 @@ def _add_missing_carriers_from_costs(n, costs, carriers): n.import_components_from_dataframe(emissions, "Carrier") -def load_costs(tech_costs, config, elec_config, Nyears=1.0): +def load_costs(tech_costs, params, elec_params, Nyears=1.0): # set all asset costs and other parameters costs = pd.read_csv(tech_costs, index_col=[0, 1]).sort_index() @@ -143,7 +143,7 @@ def load_costs(tech_costs, config, elec_config, Nyears=1.0): costs.loc[costs.unit.str.contains("/kW"), "value"] *= 1e3 costs.unit = costs.unit.str.replace("/kW", "/MW") - fill_values = config["fill_values"] + fill_values = params["fill_values"] costs = costs.value.unstack().fillna(fill_values) costs["capital_cost"] = ( @@ -166,8 +166,8 @@ def load_costs(tech_costs, config, elec_config, Nyears=1.0): costs.at["CCGT", "co2_emissions"] = costs.at["gas", "co2_emissions"] costs.at["solar", "capital_cost"] = ( - config["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"] - + (1 - config["rooftop_share"]) * costs.at["solar-utility", "capital_cost"] + params["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"] + + (1 - params["rooftop_share"]) * costs.at["solar-utility", "capital_cost"] ) def costs_for_storage(store, link1, link2=None, max_hours=1.0): @@ -178,7 +178,7 @@ def load_costs(tech_costs, config, elec_config, Nyears=1.0): dict(capital_cost=capital_cost, marginal_cost=0.0, co2_emissions=0.0) ) - max_hours = elec_config["max_hours"] + max_hours = elec_params["max_hours"] costs.loc["battery"] = costs_for_storage( costs.loc["battery storage"], costs.loc["battery inverter"], @@ -192,7 +192,7 @@ def load_costs(tech_costs, config, elec_config, Nyears=1.0): ) for attr in ("marginal_cost", "capital_cost"): - overwrites = config.get(attr) + overwrites = params.get(attr) if overwrites is not None: overwrites = pd.Series(overwrites) costs.loc[overwrites.index, attr] = overwrites @@ -356,7 +356,7 @@ def attach_conventional_generators( ppl, conventional_carriers, extendable_carriers, - conventional_config, + conventional_params, conventional_inputs, ): carriers = set(conventional_carriers) | set(extendable_carriers["Generator"]) @@ -393,12 +393,12 @@ def attach_conventional_generators( lifetime=(ppl.dateout - ppl.datein).fillna(np.inf), ) - for carrier in conventional_config: + for carrier in conventional_params: # Generators with technology affected idx = n.generators.query("carrier == @carrier").index - for attr in list(set(conventional_config[carrier]) & set(n.generators)): - values = conventional_config[carrier][attr] + for attr in list(set(conventional_params[carrier]) & set(n.generators)): + values = conventional_params[carrier][attr] if f"conventional_{carrier}_{attr}" in conventional_inputs: # Values affecting generators of technology k country-specific @@ -413,7 +413,7 @@ def attach_conventional_generators( n.generators.loc[idx, attr] = values -def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **config): +def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **params): _add_missing_carriers_from_costs(n, costs, carriers) ppl = ( @@ -468,9 +468,9 @@ def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **con ) if "PHS" in carriers and not phs.empty: - # fill missing max hours to config value and + # fill missing max hours to params value and # assume no natural inflow due to lack of data - max_hours = config.get("PHS_max_hours", 6) + max_hours = params.get("PHS_max_hours", 6) phs = phs.replace({"max_hours": {0: max_hours}}) n.madd( "StorageUnit", @@ -486,7 +486,7 @@ def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **con ) if "hydro" in carriers and not hydro.empty: - hydro_max_hours = config.get("hydro_max_hours") + hydro_max_hours = params.get("hydro_max_hours") assert hydro_max_hours is not None, "No path for hydro capacities given." @@ -636,13 +636,12 @@ def attach_OPSD_renewables(n, tech_map): n.generators.p_nom_min.update(gens.bus.map(caps).dropna()) -def estimate_renewable_capacities(n, config): - year = config["electricity"]["estimate_renewable_capacities"]["year"] - tech_map = config["electricity"]["estimate_renewable_capacities"][ +def estimate_renewable_capacities(n, electricity_params, countries): + year = electricity_params["estimate_renewable_capacities"]["year"] + tech_map = electricity_params["estimate_renewable_capacities"][ "technology_mapping" ] - countries = config["countries"] - expansion_limit = config["electricity"]["estimate_renewable_capacities"][ + expansion_limit = electricity_params["estimate_renewable_capacities"][ "expansion_limit" ] @@ -759,7 +758,7 @@ if __name__ == "__main__": ppl, conventional_carriers, extendable_carriers, - snakemake.config.get("conventional", {}), + snakemake.params.get("conventional", {}), conventional_inputs, ) @@ -773,15 +772,15 @@ if __name__ == "__main__": ) if "hydro" in renewable_carriers: - conf = snakemake.params["renewable"]["hydro"] + para = snakemake.params["renewable"]["hydro"] attach_hydro( n, costs, ppl, snakemake.input.profile_hydro, snakemake.input.hydro_capacities, - conf.pop("carriers", []), - **conf, + para.pop("carriers", []), + **para, ) if "estimate_renewable_capacities" not in snakemake.params["electricity"]: @@ -829,7 +828,7 @@ if __name__ == "__main__": "technology_mapping" ] attach_OPSD_renewables(n, tech_map) - estimate_renewable_capacities(n, snakemake.config) + estimate_renewable_capacities(n, snakemake.params["electricity"],snakemake.params["countries"]) update_p_nom_max(n) diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index 35218e2c..0b423ad5 100644 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -210,9 +210,9 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_biomass_potentials", simpl="", clusters="5") - config = snakemake.params["biomass"] - year = config["year"] - scenario = config["scenario"] + params = snakemake.params["biomass"] + year = params["year"] + scenario = params["scenario"] enspreso = enspreso_biomass_potentials(year, scenario) @@ -228,7 +228,7 @@ if __name__ == "__main__": df.to_csv(snakemake.output.biomass_potentials_all) - grouper = {v: k for k, vv in config["classes"].items() for v in vv} + grouper = {v: k for k, vv in params["classes"].items() for v in vv} df = df.groupby(grouper, axis=1).sum() df *= 1e6 # TWh/a to MWh/a diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 5099d140..3e3cb485 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -737,7 +737,7 @@ if __name__ == "__main__": logging.basicConfig(level=snakemake.config["logging"]["level"]) - config = snakemake.params["energy"] + params = snakemake.params["energy"] nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index("index") population = nuts3["pop"].groupby(nuts3.country).sum() @@ -745,7 +745,7 @@ if __name__ == "__main__": countries = snakemake.params["countries"] idees_countries = pd.Index(countries).intersection(eu28) - data_year = config["energy_totals_year"] + data_year = params["energy_totals_year"] report_year = snakemake.params["energy"]["eurostat_report_year"] input_eurostat = snakemake.input.eurostat eurostat = build_eurostat(input_eurostat, countries, report_year, data_year) @@ -755,7 +755,7 @@ if __name__ == "__main__": energy = build_energy_totals(countries, eurostat, swiss, idees) energy.to_csv(snakemake.output.energy_name) - base_year_emissions = config["base_emissions_year"] + base_year_emissions = params["base_emissions_year"] emissions_scope = snakemake.params["energy"]["emissions"] eea_co2 = build_eea_co2(snakemake.input.co2, base_year_emissions, emissions_scope) eurostat_co2 = build_eurostat_co2( diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 5453ac5c..dd686be3 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -130,7 +130,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_hydro_profile") configure_logging(snakemake) - config_hydro = snakemake.params["renewable"]["hydro"] + params_hydro = snakemake.params["renewable"]["hydro"] cutout = atlite.Cutout(snakemake.input.cutout) countries = snakemake.params["countries"] @@ -151,7 +151,7 @@ if __name__ == "__main__": normalize_using_yearly=eia_stats, ) - if "clip_min_inflow" in config_hydro: - inflow = inflow.where(inflow > config_hydro["clip_min_inflow"], 0) + if "clip_min_inflow" in params_hydro: + inflow = inflow.where(inflow > params_hydro["clip_min_inflow"], 0) inflow.to_netcdf(snakemake.output[0]) diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index c28351c1..9f8c47d0 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -101,8 +101,8 @@ def add_ammonia_energy_demand(demand): def get_ammonia_by_fuel(x): fuels = { - "gas": config["MWh_CH4_per_tNH3_SMR"], - "electricity": config["MWh_elec_per_tNH3_SMR"], + "gas": params["MWh_CH4_per_tNH3_SMR"], + "electricity": params["MWh_elec_per_tNH3_SMR"], } return pd.Series({k: x * v for k, v in fuels.items()}) @@ -112,7 +112,7 @@ def add_ammonia_energy_demand(demand): index=demand.index, fill_value=0.0 ) - ammonia = pd.DataFrame({"ammonia": ammonia * config["MWh_NH3_per_tNH3"]}).T + ammonia = pd.DataFrame({"ammonia": ammonia * params["MWh_NH3_per_tNH3"]}).T demand["Ammonia"] = ammonia.unstack().reindex(index=demand.index, fill_value=0.0) @@ -178,8 +178,8 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_industrial_energy_demand_per_country_today") - config = snakemake.params["industry"] - year = config.get("reference_year", 2015) + params = snakemake.params["industry"] + year = params.get("reference_year", 2015) countries = pd.Index(snakemake.params["countries"]) demand = industrial_energy_demand(countries.intersection(eu28), year) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 62073ea1..889c9ecd 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -264,9 +264,9 @@ def separate_basic_chemicals(demand, year): # assume HVC, methanol, chlorine production proportional to non-ammonia basic chemicals distribution_key = demand["Basic chemicals"] / demand["Basic chemicals"].sum() - demand["HVC"] = config["HVC_production_today"] * 1e3 * distribution_key - demand["Chlorine"] = config["chlorine_production_today"] * 1e3 * distribution_key - demand["Methanol"] = config["methanol_production_today"] * 1e3 * distribution_key + demand["HVC"] = params["HVC_production_today"] * 1e3 * distribution_key + demand["Chlorine"] = params["chlorine_production_today"] * 1e3 * distribution_key + demand["Methanol"] = params["methanol_production_today"] * 1e3 * distribution_key demand.drop(columns=["Basic chemicals"], inplace=True) @@ -283,7 +283,7 @@ if __name__ == "__main__": year = snakemake.params["industry"]["reference_year"] - config = snakemake.params["industry"] + params = snakemake.params["industry"] jrc_dir = snakemake.input.jrc eurostat_dir = snakemake.input.eurostat diff --git a/scripts/build_industrial_production_per_country_tomorrow.py b/scripts/build_industrial_production_per_country_tomorrow.py index b9ac9b16..609170aa 100644 --- a/scripts/build_industrial_production_per_country_tomorrow.py +++ b/scripts/build_industrial_production_per_country_tomorrow.py @@ -15,7 +15,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_industrial_production_per_country_tomorrow") - config = snakemake.params["industry"] + params = snakemake.params["industry"] investment_year = int(snakemake.wildcards.planning_horizons) @@ -25,8 +25,8 @@ if __name__ == "__main__": keys = ["Integrated steelworks", "Electric arc"] total_steel = production[keys].sum(axis=1) - st_primary_fraction = get(config["St_primary_fraction"], investment_year) - dri_fraction = get(config["DRI_fraction"], investment_year) + st_primary_fraction = get(params["St_primary_fraction"], investment_year) + dri_fraction = get(params["DRI_fraction"], investment_year) int_steel = production["Integrated steelworks"].sum() fraction_persistent_primary = st_primary_fraction * total_steel.sum() / int_steel @@ -51,7 +51,7 @@ if __name__ == "__main__": key_pri = "Aluminium - primary production" key_sec = "Aluminium - secondary production" - al_primary_fraction = get(config["Al_primary_fraction"], investment_year) + al_primary_fraction = get(params["Al_primary_fraction"], investment_year) fraction_persistent_primary = ( al_primary_fraction * total_aluminium.sum() / production[key_pri].sum() ) @@ -60,15 +60,15 @@ if __name__ == "__main__": production[key_sec] = total_aluminium - production[key_pri] production["HVC (mechanical recycling)"] = ( - get(config["HVC_mechanical_recycling_fraction"], investment_year) + get(params["HVC_mechanical_recycling_fraction"], investment_year) * production["HVC"] ) production["HVC (chemical recycling)"] = ( - get(config["HVC_chemical_recycling_fraction"], investment_year) + get(params["HVC_chemical_recycling_fraction"], investment_year) * production["HVC"] ) - production["HVC"] *= get(config["HVC_primary_fraction"], investment_year) + production["HVC"] *= get(params["HVC_primary_fraction"], investment_year) fn = snakemake.output.industrial_production_per_country_tomorrow production.to_csv(fn, float_format="%.2f") diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index c3e2bd49..2ec007a9 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -185,10 +185,10 @@ def iron_and_steel(): df[sector] = df["Electric arc"] # add H2 consumption for DRI at 1.7 MWh H2 /ton steel - df.at["hydrogen", sector] = config["H2_DRI"] + df.at["hydrogen", sector] = params["H2_DRI"] # add electricity consumption in DRI shaft (0.322 MWh/tSl) - df.at["elec", sector] += config["elec_DRI"] + df.at["elec", sector] += params["elec_DRI"] ## Integrated steelworks # could be used in combination with CCS) @@ -383,19 +383,19 @@ def chemicals_industry(): assert s_emi.index[0] == sector # convert from MtHVC/a to ktHVC/a - s_out = config["HVC_production_today"] * 1e3 + s_out = params["HVC_production_today"] * 1e3 # tCO2/t material df.loc["process emission", sector] += ( s_emi["Process emissions"] - - config["petrochemical_process_emissions"] * 1e3 - - config["NH3_process_emissions"] * 1e3 + - params["petrochemical_process_emissions"] * 1e3 + - params["NH3_process_emissions"] * 1e3 ) / s_out # emissions originating from feedstock, could be non-fossil origin # tCO2/t material df.loc["process emission from feedstock", sector] += ( - config["petrochemical_process_emissions"] * 1e3 + params["petrochemical_process_emissions"] * 1e3 ) / s_out # convert from ktoe/a to GWh/a @@ -405,18 +405,18 @@ def chemicals_industry(): # subtract ammonia energy demand (in ktNH3/a) ammonia = pd.read_csv(snakemake.input.ammonia_production, index_col=0) ammonia_total = ammonia.loc[ammonia.index.intersection(eu28), str(year)].sum() - df.loc["methane", sector] -= ammonia_total * config["MWh_CH4_per_tNH3_SMR"] - df.loc["elec", sector] -= ammonia_total * config["MWh_elec_per_tNH3_SMR"] + df.loc["methane", sector] -= ammonia_total * params["MWh_CH4_per_tNH3_SMR"] + df.loc["elec", sector] -= ammonia_total * params["MWh_elec_per_tNH3_SMR"] # subtract chlorine demand - chlorine_total = config["chlorine_production_today"] - df.loc["hydrogen", sector] -= chlorine_total * config["MWh_H2_per_tCl"] - df.loc["elec", sector] -= chlorine_total * config["MWh_elec_per_tCl"] + chlorine_total = params["chlorine_production_today"] + df.loc["hydrogen", sector] -= chlorine_total * params["MWh_H2_per_tCl"] + df.loc["elec", sector] -= chlorine_total * params["MWh_elec_per_tCl"] # subtract methanol demand - methanol_total = config["methanol_production_today"] - df.loc["methane", sector] -= methanol_total * config["MWh_CH4_per_tMeOH"] - df.loc["elec", sector] -= methanol_total * config["MWh_elec_per_tMeOH"] + methanol_total = params["methanol_production_today"] + df.loc["methane", sector] -= methanol_total * params["MWh_CH4_per_tMeOH"] + df.loc["elec", sector] -= methanol_total * params["MWh_elec_per_tMeOH"] # MWh/t material df.loc[sources, sector] = df.loc[sources, sector] / s_out @@ -427,37 +427,37 @@ def chemicals_industry(): sector = "HVC (mechanical recycling)" df[sector] = 0.0 - df.loc["elec", sector] = config["MWh_elec_per_tHVC_mechanical_recycling"] + df.loc["elec", sector] = params["MWh_elec_per_tHVC_mechanical_recycling"] # HVC chemical recycling sector = "HVC (chemical recycling)" df[sector] = 0.0 - df.loc["elec", sector] = config["MWh_elec_per_tHVC_chemical_recycling"] + df.loc["elec", sector] = params["MWh_elec_per_tHVC_chemical_recycling"] # Ammonia sector = "Ammonia" df[sector] = 0.0 if snakemake.params["sector"].get("ammonia", False): - df.loc["ammonia", sector] = config["MWh_NH3_per_tNH3"] + df.loc["ammonia", sector] = params["MWh_NH3_per_tNH3"] else: - df.loc["hydrogen", sector] = config["MWh_H2_per_tNH3_electrolysis"] - df.loc["elec", sector] = config["MWh_elec_per_tNH3_electrolysis"] + df.loc["hydrogen", sector] = params["MWh_H2_per_tNH3_electrolysis"] + df.loc["elec", sector] = params["MWh_elec_per_tNH3_electrolysis"] # Chlorine sector = "Chlorine" df[sector] = 0.0 - df.loc["hydrogen", sector] = config["MWh_H2_per_tCl"] - df.loc["elec", sector] = config["MWh_elec_per_tCl"] + df.loc["hydrogen", sector] = params["MWh_H2_per_tCl"] + df.loc["elec", sector] = params["MWh_elec_per_tCl"] # Methanol sector = "Methanol" df[sector] = 0.0 - df.loc["methane", sector] = config["MWh_CH4_per_tMeOH"] - df.loc["elec", sector] = config["MWh_elec_per_tMeOH"] + df.loc["methane", sector] = params["MWh_CH4_per_tMeOH"] + df.loc["elec", sector] = params["MWh_elec_per_tMeOH"] # Other chemicals @@ -1465,10 +1465,10 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_industry_sector_ratios") - # TODO make config option + # TODO make params option year = 2015 - config = snakemake.params["industry"] + params = snakemake.params["industry"] df = pd.concat( [ diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 8fba74e6..91463c23 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -64,7 +64,7 @@ Inputs - ``resources/offshore_shapes.geojson``: confer :ref:`shapes` - ``resources/regions_onshore.geojson``: (if not offshore wind), confer :ref:`busregions` - ``resources/regions_offshore.geojson``: (if offshore wind), :ref:`busregions` -- ``"cutouts/" + config["renewable"][{technology}]['cutout']``: :ref:`cutout` +- ``"cutouts/" + params["renewable"][{technology}]['cutout']``: :ref:`cutout` - ``networks/base.nc``: :ref:`base` Outputs @@ -204,14 +204,14 @@ if __name__ == "__main__": nprocesses = int(snakemake.threads) noprogress = snakemake.config["run"].get("disable_progressbar", True) - config = snakemake.params["renewable"][snakemake.wildcards.technology] - resource = config["resource"] # pv panel config / wind turbine config - correction_factor = config.get("correction_factor", 1.0) - capacity_per_sqkm = config["capacity_per_sqkm"] - p_nom_max_meth = config.get("potential", "conservative") + params = snakemake.params["renewable"][snakemake.wildcards.technology] + resource = params["resource"] # pv panel params / wind turbine params + correction_factor = params.get("correction_factor", 1.0) + capacity_per_sqkm = params["capacity_per_sqkm"] + p_nom_max_meth = params.get("potential", "conservative") - if isinstance(config.get("corine", {}), list): - config["corine"] = {"grid_codes": config["corine"]} + if isinstance(params.get("corine", {}), list): + params["corine"] = {"grid_codes": params["corine"]} if correction_factor != 1.0: logger.info(f"correction_factor is set as {correction_factor}") @@ -229,13 +229,13 @@ if __name__ == "__main__": regions = regions.set_index("name").rename_axis("bus") buses = regions.index - res = config.get("excluder_resolution", 100) + res = params.get("excluder_resolution", 100) excluder = atlite.ExclusionContainer(crs=3035, res=res) - if config["natura"]: + if params["natura"]: excluder.add_raster(snakemake.input.natura, nodata=0, allow_no_overlap=True) - corine = config.get("corine", {}) + corine = params.get("corine", {}) if "grid_codes" in corine: codes = corine["grid_codes"] excluder.add_raster(snakemake.input.corine, codes=codes, invert=True, crs=3035) @@ -246,28 +246,28 @@ if __name__ == "__main__": snakemake.input.corine, codes=codes, buffer=buffer, crs=3035 ) - if "ship_threshold" in config: + if "ship_threshold" in params: shipping_threshold = ( - config["ship_threshold"] * 8760 * 6 + params["ship_threshold"] * 8760 * 6 ) # approximation because 6 years of data which is hourly collected func = functools.partial(np.less, shipping_threshold) excluder.add_raster( snakemake.input.ship_density, codes=func, crs=4326, allow_no_overlap=True ) - if config.get("max_depth"): + if params.get("max_depth"): # lambda not supported for atlite + multiprocessing # use named function np.greater with partially frozen argument instead # and exclude areas where: -max_depth > grid cell depth - func = functools.partial(np.greater, -config["max_depth"]) + func = functools.partial(np.greater, -params["max_depth"]) excluder.add_raster(snakemake.input.gebco, codes=func, crs=4326, nodata=-1000) - if "min_shore_distance" in config: - buffer = config["min_shore_distance"] + if "min_shore_distance" in params: + buffer = params["min_shore_distance"] excluder.add_geometry(snakemake.input.country_shapes, buffer=buffer) - if "max_shore_distance" in config: - buffer = config["max_shore_distance"] + if "max_shore_distance" in params: + buffer = params["max_shore_distance"] excluder.add_geometry( snakemake.input.country_shapes, buffer=buffer, invert=True ) @@ -309,7 +309,7 @@ if __name__ == "__main__": p_nom_max = capacities / max_cap_factor else: raise AssertionError( - 'Config key `potential` should be one of "simple" ' + 'params key `potential` should be one of "simple" ' f'(default) or "conservative", not "{p_nom_max_meth}"' ) @@ -358,13 +358,13 @@ if __name__ == "__main__": # select only buses with some capacity and minimal capacity factor ds = ds.sel( bus=( - (ds["profile"].mean("time") > config.get("min_p_max_pu", 0.0)) - & (ds["p_nom_max"] > config.get("min_p_nom_max", 0.0)) + (ds["profile"].mean("time") > params.get("min_p_max_pu", 0.0)) + & (ds["p_nom_max"] > params.get("min_p_nom_max", 0.0)) ) ) - if "clip_p_max_pu" in config: - min_p_max_pu = config["clip_p_max_pu"] + if "clip_p_max_pu" in params: + min_p_max_pu = params["clip_p_max_pu"] ds["profile"] = ds["profile"].where(ds["profile"] >= min_p_max_pu, 0) ds.to_netcdf(snakemake.output.profile) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 86a9bc82..1db9b916 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -727,7 +727,7 @@ def cycling_shift(df, steps=1): return df -def prepare_costs(cost_file, config, nyears): +def prepare_costs(cost_file, params, nyears): # set all asset costs and other parameters costs = pd.read_csv(cost_file, index_col=[0, 1]).sort_index() @@ -739,7 +739,7 @@ def prepare_costs(cost_file, config, nyears): costs.loc[:, "value"].unstack(level=1).groupby("technology").sum(min_count=1) ) - costs = costs.fillna(config["fill_values"]) + costs = costs.fillna(params["fill_values"]) def annuity_factor(v): return annuity(v["lifetime"], v["discount rate"]) + v["FOM"] / 100 From 6366dc4c55076c084caddcbd67f997c17e9360bd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 17 May 2023 17:26:00 +0000 Subject: [PATCH 17/41] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_electricity.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index a3e033e7..adb8bf31 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -638,9 +638,7 @@ def attach_OPSD_renewables(n, tech_map): def estimate_renewable_capacities(n, electricity_params, countries): year = electricity_params["estimate_renewable_capacities"]["year"] - tech_map = electricity_params["estimate_renewable_capacities"][ - "technology_mapping" - ] + tech_map = electricity_params["estimate_renewable_capacities"]["technology_mapping"] expansion_limit = electricity_params["estimate_renewable_capacities"][ "expansion_limit" ] @@ -828,7 +826,9 @@ if __name__ == "__main__": "technology_mapping" ] attach_OPSD_renewables(n, tech_map) - estimate_renewable_capacities(n, snakemake.params["electricity"],snakemake.params["countries"]) + estimate_renewable_capacities( + n, snakemake.params["electricity"], snakemake.params["countries"] + ) update_p_nom_max(n) From d97a5434630d93c2f41ab64c8304ccc94961bee9 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Wed, 17 May 2023 19:37:47 +0200 Subject: [PATCH 18/41] missing a coma in build_electricity.smk --- rules/build_electricity.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 88e7c548..1ba1a1c5 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -277,7 +277,7 @@ rule add_electricity: countries=config["countries"], renewable=config["renewable"], electricity=config["electricity"], - conventional=config.get("conventional", {}) + conventional=config.get("conventional", {}), costs=config["costs"], input: **{ From c043100ada790d088ea190e9f8ba634e28a204e1 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Fri, 19 May 2023 15:42:07 +0200 Subject: [PATCH 19/41] add more params in simplify_network.py --- rules/build_electricity.smk | 8 +++- scripts/add_electricity.py | 2 +- scripts/cluster_network.py | 2 +- scripts/simplify_network.py | 76 +++++++++++++++++++------------------ 4 files changed, 49 insertions(+), 39 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 1ba1a1c5..f33d1686 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -318,6 +318,12 @@ rule simplify_network: clustering=config["clustering"], electricity=config["electricity"], costs=config["costs"], + renewable=config["renewable"], + length_factor=config["lines"]["length_factor"], + p_max_pu=config["links"].get("p_max_pu", 1.0), + exclude_carriers=config["clustering"]["simplify_network"].get("exclude_carriers", []), + focus_weights=config.get("focus_weights", None), + solver_name=config["solving"]["solver"]["name"], input: network=RESOURCES + "networks/elec.nc", tech_costs=COSTS, @@ -350,7 +356,7 @@ rule cluster_network: lines=config["lines"], renewable=config["renewable"], clustering=config["clustering"], - enable=config["enable"].get("custom_busmap", False), + custom_busmap=config["enable"].get("custom_busmap", False), input: network=RESOURCES + "networks/elec_s{simpl}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson", diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index adb8bf31..3717f0e8 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -756,7 +756,7 @@ if __name__ == "__main__": ppl, conventional_carriers, extendable_carriers, - snakemake.params.get("conventional", {}), + snakemake.params["conventional"], conventional_inputs, ) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 78798804..d217e5ed 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -521,7 +521,7 @@ if __name__ == "__main__": for p in aggregation_strategies.keys() } - custom_busmap = snakemake.params["enable"].get("custom_busmap", False) + custom_busmap = snakemake.params["custom_busmap"] if custom_busmap: custom_busmap = pd.read_csv( snakemake.input.custom_busmap, index_col=0, squeeze=True diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index bb7f615c..91dde6b1 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -149,17 +149,17 @@ def simplify_network_to_380(n): return n, trafo_map -def _prepare_connection_costs_per_link(n, costs, config): +def _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param): if n.links.empty: return {} connection_costs_per_link = {} - for tech in config["renewable"]: + for tech in renewable_param: if tech.startswith("offwind"): connection_costs_per_link[tech] = ( n.links.length - * config["lines"]["length_factor"] + * length_factor_param * ( n.links.underwater_fraction * costs.at[tech + "-connection-submarine", "capital_cost"] @@ -172,10 +172,10 @@ def _prepare_connection_costs_per_link(n, costs, config): def _compute_connection_costs_to_bus( - n, busmap, costs, config, connection_costs_per_link=None, buses=None + n, busmap, costs, renewable_param, length_factor_param, connection_costs_per_link=None, buses=None ): if connection_costs_per_link is None: - connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, config) + connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param) if buses is None: buses = busmap.index[busmap.index != busmap.values] @@ -265,7 +265,7 @@ def _aggregate_and_move_components( n.mremove(c, df.index[df.bus0.isin(buses_to_del) | df.bus1.isin(buses_to_del)]) -def simplify_links(n, costs, config, output, aggregation_strategies=dict()): +def simplify_links(n, costs, renewable_param, length_factor_param, p_max_pu_param, exclude_carriers_param, output, aggregation_strategies=dict()): ## Complex multi-node links are folded into end-points logger.info("Simplifying connected link components") @@ -315,7 +315,7 @@ def simplify_links(n, costs, config, output, aggregation_strategies=dict()): busmap = n.buses.index.to_series() - connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, config) + connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param) connection_costs_to_bus = pd.DataFrame( 0.0, index=n.buses.index, columns=list(connection_costs_per_link) ) @@ -333,12 +333,11 @@ def simplify_links(n, costs, config, output, aggregation_strategies=dict()): ) busmap.loc[buses] = b[np.r_[0, m.argmin(axis=0), 1]] connection_costs_to_bus.loc[buses] += _compute_connection_costs_to_bus( - n, busmap, costs, config, connection_costs_per_link, buses + n, busmap, costs, renewable_param, length_factor_param, connection_costs_per_link, buses ) all_links = [i for _, i in sum(links, [])] - p_max_pu = config["links"].get("p_max_pu", 1.0) lengths = n.links.loc[all_links, "length"] name = lengths.idxmax() + "+{}".format(len(links) - 1) params = dict( @@ -354,8 +353,8 @@ def simplify_links(n, costs, config, output, aggregation_strategies=dict()): / lengths.sum() * n.links.loc[all_links, "underwater_fraction"] ), - p_max_pu=p_max_pu, - p_min_pu=-p_max_pu, + p_max_pu=p_max_pu_param, + p_min_pu=-p_max_pu_param, underground=False, under_construction=False, ) @@ -377,33 +376,29 @@ def simplify_links(n, costs, config, output, aggregation_strategies=dict()): logger.debug("Collecting all components using the busmap") - exclude_carriers = config["clustering"]["simplify_network"].get( - "exclude_carriers", [] - ) - _aggregate_and_move_components( n, busmap, connection_costs_to_bus, output, aggregation_strategies=aggregation_strategies, - exclude_carriers=exclude_carriers, + exclude_carriers=exclude_carriers_param, ) return n, busmap -def remove_stubs(n, costs, config, output, aggregation_strategies=dict()): +def remove_stubs(n, costs, renewable_param, length_factor_param, clustering_param, exclude_carriers_param, output, aggregation_strategies=dict()): logger.info("Removing stubs") - across_borders = config["clustering"]["simplify_network"].get( + across_borders = clustering_param["simplify_network"].get( "remove_stubs_across_borders", True ) matching_attrs = [] if across_borders else ["country"] busmap = busmap_by_stubs(n, matching_attrs) - connection_costs_to_bus = _compute_connection_costs_to_bus(n, busmap, costs, config) + connection_costs_to_bus = _compute_connection_costs_to_bus(n, busmap, costs, renewable_param, length_factor_param) - exclude_carriers = config["clustering"]["simplify_network"].get( + exclude_carriers = clustering_param["simplify_network"].get( "exclude_carriers", [] ) @@ -473,17 +468,15 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None): def cluster( - n, n_clusters, config, algorithm="hac", feature=None, aggregation_strategies=dict() + n, n_clusters, focus_weights_param, renewable_param, solver_name_param, algorithm="hac", feature=None, aggregation_strategies=dict() ): logger.info(f"Clustering to {n_clusters} buses") - focus_weights = config.get("focus_weights", None) - renewable_carriers = pd.Index( [ tech for tech in n.generators.carrier.unique() - if tech.split("-", 2)[0] in config["renewable"] + if tech.split("-", 2)[0] in renewable_param ] ) @@ -492,10 +485,10 @@ def cluster( n_clusters, custom_busmap=False, aggregation_strategies=aggregation_strategies, - solver_name=config["solving"]["solver"]["name"], + solver_name=solver_name_param, algorithm=algorithm, feature=feature, - focus_weights=focus_weights, + focus_weights=focus_weights_param, ) return clustering.network, clustering.busmap @@ -531,23 +524,32 @@ if __name__ == "__main__": ) n, simplify_links_map = simplify_links( - n, technology_costs, snakemake.config, snakemake.output, aggregation_strategies + n, technology_costs, + snakemake.params['renewable'], + snakemake.params['length_factor'], + snakemake.params['p_max_pu'], + snakemake.params['exclude_carriers'], + snakemake.output, + aggregation_strategies ) busmaps = [trafo_map, simplify_links_map] - cluster_config = snakemake.params["clustering"]["simplify_network"] - if cluster_config.get("remove_stubs", True): + cluster_param = snakemake.params["clustering"]["simplify_network"] + if cluster_param.get("remove_stubs", True): n, stub_map = remove_stubs( n, technology_costs, - snakemake.config, + snakemake.params['renewable'], + snakemake.params['length_factor'], + snakemake.params["clustering"], + snakemake.params['exclude_carriers'], snakemake.output, aggregation_strategies=aggregation_strategies, ) busmaps.append(stub_map) - if cluster_config.get("to_substations", False): + if cluster_param.get("to_substations", False): n, substation_map = aggregate_to_substations(n, aggregation_strategies) busmaps.append(substation_map) @@ -558,10 +560,10 @@ if __name__ == "__main__": .get("cluster_network", {}) .get("algorithm", "hac") == "hac" - or cluster_config.get("algorithm", "hac") == "hac" + or cluster_param.get("algorithm", "hac") == "hac" ): carriers = ( - cluster_config.get("feature", "solar+onwind-time").split("-")[0].split("+") + cluster_param.get("feature", "solar+onwind-time").split("-")[0].split("+") ) for carrier in carriers: buses_i = list( @@ -577,9 +579,11 @@ if __name__ == "__main__": n, cluster_map = cluster( n, int(snakemake.wildcards.simpl), - snakemake.config, - cluster_config.get("algorithm", "hac"), - cluster_config.get("feature", None), + snakemake.params['focus_weights'], + snakemake.params['renewable'], + snakemake.params['solver_name'], + cluster_param.get("algorithm", "hac"), + cluster_param.get("feature", None), aggregation_strategies, ) busmaps.append(cluster_map) From 216a02fba12fe445e36fae1c019820715379c437 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 19 May 2023 13:42:43 +0000 Subject: [PATCH 20/41] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_electricity.smk | 4 +- scripts/simplify_network.py | 90 +++++++++++++++++++++++++++---------- 2 files changed, 69 insertions(+), 25 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index f33d1686..45436c5d 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -321,7 +321,9 @@ rule simplify_network: renewable=config["renewable"], length_factor=config["lines"]["length_factor"], p_max_pu=config["links"].get("p_max_pu", 1.0), - exclude_carriers=config["clustering"]["simplify_network"].get("exclude_carriers", []), + exclude_carriers=config["clustering"]["simplify_network"].get( + "exclude_carriers", [] + ), focus_weights=config.get("focus_weights", None), solver_name=config["solving"]["solver"]["name"], input: diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 91dde6b1..689e8084 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -172,10 +172,18 @@ def _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_ def _compute_connection_costs_to_bus( - n, busmap, costs, renewable_param, length_factor_param, connection_costs_per_link=None, buses=None + n, + busmap, + costs, + renewable_param, + length_factor_param, + connection_costs_per_link=None, + buses=None, ): if connection_costs_per_link is None: - connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param) + connection_costs_per_link = _prepare_connection_costs_per_link( + n, costs, renewable_param, length_factor_param + ) if buses is None: buses = busmap.index[busmap.index != busmap.values] @@ -265,7 +273,16 @@ def _aggregate_and_move_components( n.mremove(c, df.index[df.bus0.isin(buses_to_del) | df.bus1.isin(buses_to_del)]) -def simplify_links(n, costs, renewable_param, length_factor_param, p_max_pu_param, exclude_carriers_param, output, aggregation_strategies=dict()): +def simplify_links( + n, + costs, + renewable_param, + length_factor_param, + p_max_pu_param, + exclude_carriers_param, + output, + aggregation_strategies=dict(), +): ## Complex multi-node links are folded into end-points logger.info("Simplifying connected link components") @@ -315,7 +332,9 @@ def simplify_links(n, costs, renewable_param, length_factor_param, p_max_pu_para busmap = n.buses.index.to_series() - connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param) + connection_costs_per_link = _prepare_connection_costs_per_link( + n, costs, renewable_param, length_factor_param + ) connection_costs_to_bus = pd.DataFrame( 0.0, index=n.buses.index, columns=list(connection_costs_per_link) ) @@ -333,7 +352,13 @@ def simplify_links(n, costs, renewable_param, length_factor_param, p_max_pu_para ) busmap.loc[buses] = b[np.r_[0, m.argmin(axis=0), 1]] connection_costs_to_bus.loc[buses] += _compute_connection_costs_to_bus( - n, busmap, costs, renewable_param, length_factor_param, connection_costs_per_link, buses + n, + busmap, + costs, + renewable_param, + length_factor_param, + connection_costs_per_link, + buses, ) all_links = [i for _, i in sum(links, [])] @@ -387,7 +412,16 @@ def simplify_links(n, costs, renewable_param, length_factor_param, p_max_pu_para return n, busmap -def remove_stubs(n, costs, renewable_param, length_factor_param, clustering_param, exclude_carriers_param, output, aggregation_strategies=dict()): +def remove_stubs( + n, + costs, + renewable_param, + length_factor_param, + clustering_param, + exclude_carriers_param, + output, + aggregation_strategies=dict(), +): logger.info("Removing stubs") across_borders = clustering_param["simplify_network"].get( @@ -396,12 +430,12 @@ def remove_stubs(n, costs, renewable_param, length_factor_param, clustering_para matching_attrs = [] if across_borders else ["country"] busmap = busmap_by_stubs(n, matching_attrs) - connection_costs_to_bus = _compute_connection_costs_to_bus(n, busmap, costs, renewable_param, length_factor_param) - - exclude_carriers = clustering_param["simplify_network"].get( - "exclude_carriers", [] + connection_costs_to_bus = _compute_connection_costs_to_bus( + n, busmap, costs, renewable_param, length_factor_param ) + exclude_carriers = clustering_param["simplify_network"].get("exclude_carriers", []) + _aggregate_and_move_components( n, busmap, @@ -468,7 +502,14 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None): def cluster( - n, n_clusters, focus_weights_param, renewable_param, solver_name_param, algorithm="hac", feature=None, aggregation_strategies=dict() + n, + n_clusters, + focus_weights_param, + renewable_param, + solver_name_param, + algorithm="hac", + feature=None, + aggregation_strategies=dict(), ): logger.info(f"Clustering to {n_clusters} buses") @@ -524,13 +565,14 @@ if __name__ == "__main__": ) n, simplify_links_map = simplify_links( - n, technology_costs, - snakemake.params['renewable'], - snakemake.params['length_factor'], - snakemake.params['p_max_pu'], - snakemake.params['exclude_carriers'], - snakemake.output, - aggregation_strategies + n, + technology_costs, + snakemake.params["renewable"], + snakemake.params["length_factor"], + snakemake.params["p_max_pu"], + snakemake.params["exclude_carriers"], + snakemake.output, + aggregation_strategies, ) busmaps = [trafo_map, simplify_links_map] @@ -540,10 +582,10 @@ if __name__ == "__main__": n, stub_map = remove_stubs( n, technology_costs, - snakemake.params['renewable'], - snakemake.params['length_factor'], + snakemake.params["renewable"], + snakemake.params["length_factor"], snakemake.params["clustering"], - snakemake.params['exclude_carriers'], + snakemake.params["exclude_carriers"], snakemake.output, aggregation_strategies=aggregation_strategies, ) @@ -579,9 +621,9 @@ if __name__ == "__main__": n, cluster_map = cluster( n, int(snakemake.wildcards.simpl), - snakemake.params['focus_weights'], - snakemake.params['renewable'], - snakemake.params['solver_name'], + snakemake.params["focus_weights"], + snakemake.params["renewable"], + snakemake.params["solver_name"], cluster_param.get("algorithm", "hac"), cluster_param.get("feature", None), aggregation_strategies, From d2f6138ca564d3286fcd8f3f93e954d22b4d16d7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 21:56:04 +0000 Subject: [PATCH 21/41] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/docformatter: v1.6.5 → v1.7.1](https://github.com/PyCQA/docformatter/compare/v1.6.5...v1.7.1) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 48915f4c..34d13746 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: # Make docstrings PEP 257 compliant - repo: https://github.com/PyCQA/docformatter - rev: v1.6.5 + rev: v1.7.1 hooks: - id: docformatter args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"] From 0f09545d9626b7e64c883a721f592847c5fd4367 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 21:56:29 +0000 Subject: [PATCH 22/41] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/_helpers.py | 9 +++++---- scripts/build_retro_cost.py | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index a67fb105..8086c1b5 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -385,10 +385,11 @@ def mock_snakemake(rulename, configfiles=[], **wildcards): def override_component_attrs(directory): - """Tell PyPSA that links can have multiple outputs by - overriding the component_attrs. This can be done for - as many buses as you need with format busi for i = 2,3,4,5,.... - See https://pypsa.org/doc/components.html#link-with-multiple-outputs-or-inputs + """ + Tell PyPSA that links can have multiple outputs by overriding the + component_attrs. This can be done for as many buses as you need with format + busi for i = 2,3,4,5,.... See https://pypsa.org/doc/components.html#link- + with-multiple-outputs-or-inputs. Parameters ---------- diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index 9dbfc375..63ae0743 100644 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -698,8 +698,8 @@ def get_solar_gains_per_year(window_area): def map_to_lstrength(l_strength, df): """ - renames column names from a pandas dataframe to map tabula retrofitting - strengths [2 = moderate, 3 = ambitious] to l_strength + Renames column names from a pandas dataframe to map tabula retrofitting + strengths [2 = moderate, 3 = ambitious] to l_strength. """ middle = len(l_strength) // 2 map_to_l = pd.MultiIndex.from_arrays( From 312dd81f216f2ab55ee2eb8b13c5a54532fe9e06 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Wed, 24 May 2023 11:13:37 +0200 Subject: [PATCH 23/41] Fix bug with underground H2 storage creation For some small model regions, none of the H2 cavern types specified in the configuration might actually be available, in which case the line `h2_caverns = h2_caverns[cavern_types].sum(axis=1)` throws an error. --- scripts/prepare_sector_network.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 012c9714..57deb5fd 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1105,7 +1105,11 @@ def add_storage_and_grids(n, costs): cavern_types = snakemake.config["sector"]["hydrogen_underground_storage_locations"] h2_caverns = pd.read_csv(snakemake.input.h2_cavern, index_col=0) - if not h2_caverns.empty and options["hydrogen_underground_storage"]: + if ( + not h2_caverns.empty + and options["hydrogen_underground_storage"] + and set(cavern_types).intersection(h2_caverns.columns) + ): h2_caverns = h2_caverns[cavern_types].sum(axis=1) # only use sites with at least 2 TWh potential From dc043d8aeff58455ad0e339485de375e8afb8d02 Mon Sep 17 00:00:00 2001 From: lucie_rc <104382956+LucieRC@users.noreply.github.com> Date: Wed, 24 May 2023 15:46:53 +0300 Subject: [PATCH 24/41] Update Discord link README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 02cc3163..79cdfa65 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ repository](https://doi.org/10.5281/zenodo.3601881). We strongly welcome anyone interested in contributing to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on GitHub. - In case of code-related **questions**, please post on [stack overflow](https://stackoverflow.com/questions/tagged/pypsa). - For non-programming related and more general questions please refer to the [mailing list](https://groups.google.com/group/pypsa). -- To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.gg/JTdvaEBb). +- To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the [discord server](https://discord.com/invite/AnuJBk23FU). - For **bugs and feature requests**, please use the [PyPSA-Eur Github Issues page](https://github.com/PyPSA/pypsa-eur/issues). # Licence From 1c73bb03426beaaaedc10df7ee83d8f036395a4d Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Sat, 27 May 2023 12:22:53 +0200 Subject: [PATCH 25/41] add params for solve_network --- rules/solve_electricity.smk | 6 ++++++ rules/solve_myopic.smk | 6 ++++++ rules/solve_overnight.smk | 6 ++++++ scripts/solve_network.py | 38 ++++++++++++++++++------------------- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 2e9c4b44..f0ce55c8 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -6,6 +6,12 @@ rule solve_network: params: solving=config["solving"], + config_parts={ + "foresight":config["foresight"], + "planning_horizons":config["scenario"]["planning_horizons"], + "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200), + "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"] + }, input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index ec7638cf..c682f780 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -84,6 +84,12 @@ ruleorder: add_existing_baseyear > add_brownfield rule solve_sector_network_myopic: params: solving=config["solving"], + config_parts={ + "foresight":config["foresight"], + "planning_horizons":config["scenario"]["planning_horizons"], + "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200), + "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"] + }, input: overrides="data/override_component_attrs", network=RESULTS diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index b657eb2b..050372bd 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -6,6 +6,12 @@ rule solve_sector_network: params: solving=config["solving"], + config_parts={ + "foresight":config["foresight"], + "planning_horizons":config["scenario"]["planning_horizons"], + "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200), + "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"] + }, input: overrides="data/override_component_attrs", network=RESULTS diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 180c83c6..d615fc85 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -44,14 +44,14 @@ pypsa.pf.logger.setLevel(logging.WARNING) from pypsa.descriptors import get_switchable_as_dense as get_as_dense -def add_land_use_constraint(n, config): +def add_land_use_constraint(n, param, config): if "m" in snakemake.wildcards.clusters: - _add_land_use_constraint_m(n, config) + _add_land_use_constraint_m(n, param, config) else: - _add_land_use_constraint(n, config) + _add_land_use_constraint(n, param) -def _add_land_use_constraint(n, config): +def _add_land_use_constraint(n, param): # warning: this will miss existing offwind which is not classed AC-DC and has carrier 'offwind' for carrier in ["solar", "onwind", "offwind-ac", "offwind-dc"]: @@ -80,10 +80,10 @@ def _add_land_use_constraint(n, config): n.generators.p_nom_max.clip(lower=0, inplace=True) -def _add_land_use_constraint_m(n, config): +def _add_land_use_constraint_m(n, param, config): # if generators clustering is lower than network clustering, land_use accounting is at generators clusters - planning_horizons = config["scenario"]["planning_horizons"] + planning_horizons = param["planning_horizons"] grouping_years = config["existing_capacities"]["grouping_years"] current_horizon = snakemake.wildcards.planning_horizons @@ -141,7 +141,7 @@ def add_co2_sequestration_limit(n, limit=200): ) -def prepare_network(n, solve_opts=None, config=None): +def prepare_network(n, solve_opts=None, config=None, param=None): if "clip_p_max_pu" in solve_opts: for df in ( n.generators_t.p_max_pu, @@ -191,11 +191,11 @@ def prepare_network(n, solve_opts=None, config=None): n.set_snapshots(n.snapshots[:nhours]) n.snapshot_weightings[:] = 8760.0 / nhours - if config["foresight"] == "myopic": - add_land_use_constraint(n, config) + if param["foresight"] == "myopic": + add_land_use_constraint(n, param, config) if n.stores.carrier.eq("co2 stored").any(): - limit = config["sector"].get("co2_sequestration_potential", 200) + limit = param["co2_sequestration_potential"] add_co2_sequestration_limit(n, limit=limit) return n @@ -222,7 +222,7 @@ def add_CCL_constraints(n, config): agg_p_nom_limits: data/agg_p_nom_minmax.csv """ agg_p_nom_minmax = pd.read_csv( - config["electricity"]["agg_p_nom_limits"], index_col=[0, 1] + config['electricity']["agg_p_nom_limits"], index_col=[0, 1] ) logger.info("Adding generation capacity constraints per carrier and country") p_nom = n.model["Generator-p_nom"] @@ -370,7 +370,7 @@ def add_SAFE_constraints(n, config): Which sets a reserve margin of 10% above the peak demand. """ peakdemand = n.loads_t.p_set.sum(axis=1).max() - margin = 1.0 + config["electricity"]["SAFE_reservemargin"] + margin = 1.0 + config['electricity']["SAFE_reservemargin"] reserve_margin = peakdemand * margin # TODO: do not take this from the plotting config! conv_techs = config["plotting"]["conv_techs"] @@ -590,13 +590,13 @@ def extra_functionality(n, snapshots): add_pipe_retrofit_constraint(n) -def solve_network(n, config, opts="", **kwargs): - set_of_options = config["solving"]["solver"]["options"] +def solve_network(n, config, solving_param, opts="", **kwargs): + set_of_options = solving_param["solver"]["options"] solver_options = ( - config["solving"]["solver_options"][set_of_options] if set_of_options else {} + solving_param["solver_options"][set_of_options] if set_of_options else {} ) - solver_name = config["solving"]["solver"]["name"] - cf_solving = config["solving"]["options"] + solver_name = solving_param["solver"]["name"] + cf_solving = solving_param["options"] track_iterations = cf_solving.get("track_iterations", False) min_iterations = cf_solving.get("min_iterations", 4) max_iterations = cf_solving.get("max_iterations", 6) @@ -675,10 +675,10 @@ if __name__ == "__main__": else: n = pypsa.Network(snakemake.input.network) - n = prepare_network(n, solve_opts, config=snakemake.config) + n = prepare_network(n, solve_opts, config=snakemake.config, param=snakemake.params["config_parts"]) n = solve_network( - n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver + n, config=snakemake.config, solving_param=snakemake.params["solving"], opts=opts, log_fn=snakemake.log.solver ) n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) From ead87afce1ab0d7179b9012f88df1089e609bc61 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 27 May 2023 10:23:28 +0000 Subject: [PATCH 26/41] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/solve_electricity.smk | 14 +++++++++----- rules/solve_myopic.smk | 14 +++++++++----- rules/solve_overnight.smk | 14 +++++++++----- scripts/solve_network.py | 14 ++++++++++---- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index f0ce55c8..30520dc1 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -7,11 +7,15 @@ rule solve_network: params: solving=config["solving"], config_parts={ - "foresight":config["foresight"], - "planning_horizons":config["scenario"]["planning_horizons"], - "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200), - "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"] - }, + "foresight": config["foresight"], + "planning_horizons": config["scenario"]["planning_horizons"], + "co2_sequestration_potential": config["sector"].get( + "co2_sequestration_potential", 200 + ), + "H2_retrofit_capacity_per_CH4": config["sector"][ + "H2_retrofit_capacity_per_CH4" + ], + }, input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index c682f780..22555f75 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -85,11 +85,15 @@ rule solve_sector_network_myopic: params: solving=config["solving"], config_parts={ - "foresight":config["foresight"], - "planning_horizons":config["scenario"]["planning_horizons"], - "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200), - "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"] - }, + "foresight": config["foresight"], + "planning_horizons": config["scenario"]["planning_horizons"], + "co2_sequestration_potential": config["sector"].get( + "co2_sequestration_potential", 200 + ), + "H2_retrofit_capacity_per_CH4": config["sector"][ + "H2_retrofit_capacity_per_CH4" + ], + }, input: overrides="data/override_component_attrs", network=RESULTS diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index 050372bd..a523f132 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -7,11 +7,15 @@ rule solve_sector_network: params: solving=config["solving"], config_parts={ - "foresight":config["foresight"], - "planning_horizons":config["scenario"]["planning_horizons"], - "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200), - "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"] - }, + "foresight": config["foresight"], + "planning_horizons": config["scenario"]["planning_horizons"], + "co2_sequestration_potential": config["sector"].get( + "co2_sequestration_potential", 200 + ), + "H2_retrofit_capacity_per_CH4": config["sector"][ + "H2_retrofit_capacity_per_CH4" + ], + }, input: overrides="data/override_component_attrs", network=RESULTS diff --git a/scripts/solve_network.py b/scripts/solve_network.py index d615fc85..6e07e340 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -222,7 +222,7 @@ def add_CCL_constraints(n, config): agg_p_nom_limits: data/agg_p_nom_minmax.csv """ agg_p_nom_minmax = pd.read_csv( - config['electricity']["agg_p_nom_limits"], index_col=[0, 1] + config["electricity"]["agg_p_nom_limits"], index_col=[0, 1] ) logger.info("Adding generation capacity constraints per carrier and country") p_nom = n.model["Generator-p_nom"] @@ -370,7 +370,7 @@ def add_SAFE_constraints(n, config): Which sets a reserve margin of 10% above the peak demand. """ peakdemand = n.loads_t.p_set.sum(axis=1).max() - margin = 1.0 + config['electricity']["SAFE_reservemargin"] + margin = 1.0 + config["electricity"]["SAFE_reservemargin"] reserve_margin = peakdemand * margin # TODO: do not take this from the plotting config! conv_techs = config["plotting"]["conv_techs"] @@ -675,10 +675,16 @@ if __name__ == "__main__": else: n = pypsa.Network(snakemake.input.network) - n = prepare_network(n, solve_opts, config=snakemake.config, param=snakemake.params["config_parts"]) + n = prepare_network( + n, solve_opts, config=snakemake.config, param=snakemake.params["config_parts"] + ) n = solve_network( - n, config=snakemake.config, solving_param=snakemake.params["solving"], opts=opts, log_fn=snakemake.log.solver + n, + config=snakemake.config, + solving_param=snakemake.params["solving"], + opts=opts, + log_fn=snakemake.log.solver, ) n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) From e581ca930c24de2c3e3fa4d0d6bdac0fbfa06b30 Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Sat, 27 May 2023 15:50:37 +0200 Subject: [PATCH 27/41] made params more specific --- rules/build_electricity.smk | 29 +++++--- rules/build_sector.smk | 72 ++++++++++++++++---- rules/postprocess.smk | 3 +- rules/solve_electricity.smk | 2 +- rules/solve_myopic.smk | 7 +- scripts/add_brownfield.py | 6 +- scripts/add_electricity.py | 11 ++- scripts/add_existing_baseyear.py | 2 +- scripts/add_extra_components.py | 23 +++---- scripts/build_cop_profiles.py | 2 +- scripts/build_cutout.py | 2 +- scripts/build_hydro_profile.py | 2 +- scripts/build_industrial_distribution_key.py | 2 +- scripts/build_industry_sector_ratios.py | 2 +- scripts/build_powerplants.py | 4 +- scripts/build_retro_cost.py | 2 +- scripts/build_sequestration_potentials.py | 2 +- scripts/cluster_network.py | 8 +-- scripts/plot_summary.py | 4 +- scripts/prepare_network.py | 10 +-- scripts/prepare_sector_network.py | 14 ++-- scripts/simplify_network.py | 2 +- scripts/solve_operations_network.py | 2 +- 23 files changed, 131 insertions(+), 82 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 45436c5d..9b3a03fb 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -39,7 +39,8 @@ rule build_electricity_demand: rule build_powerplants: params: - electricity=config["electricity"], + powerplants_filter=config["electricity"]["powerplants_filter"], + custom_powerplants=config["electricity"]["custom_powerplants"], countries=config["countries"], input: base_network=RESOURCES + "networks/base.nc", @@ -138,7 +139,7 @@ if config["enable"].get("build_cutout", False): rule build_cutout: params: snapshots=config["snapshots"], - atlite=config["atlite"], + cutouts=config["atlite"]["cutouts"], input: regions_onshore=RESOURCES + "regions_onshore.geojson", regions_offshore=RESOURCES + "regions_offshore.geojson", @@ -252,6 +253,7 @@ rule build_renewable_profiles: rule build_hydro_profile: params: + hydro=config["renewable"]["hydro"], countries=config["countries"], renewable=config["renewable"], input: @@ -272,8 +274,8 @@ rule build_hydro_profile: rule add_electricity: params: - lines=config["lines"], - load=config["load"], + length_factor=config["lines"]["length_factor"], + scaling_factor=config["load"]["scaling_factor"], countries=config["countries"], renewable=config["renewable"], electricity=config["electricity"], @@ -316,7 +318,7 @@ rule add_electricity: rule simplify_network: params: clustering=config["clustering"], - electricity=config["electricity"], + max_hours=config["electricity"]["max_hours"], costs=config["costs"], renewable=config["renewable"], length_factor=config["lines"]["length_factor"], @@ -352,10 +354,11 @@ rule simplify_network: rule cluster_network: params: - solving=config["solving"], - electricity=config["electricity"], + solver_name=config["solving"]["solver"]["name"], + max_hours=config["electricity"]["max_hours"], + conventional_carriers=config["electricity"].get("conventional_carriers", []), costs=config["costs"], - lines=config["lines"], + length_factor=config["lines"]["length_factor"], renewable=config["renewable"], clustering=config["clustering"], custom_busmap=config["enable"].get("custom_busmap", False), @@ -392,7 +395,8 @@ rule cluster_network: rule add_extra_components: params: costs=config["costs"], - electricity=config["electricity"], + ext_carriers=config["electricity"]["extendable_carriers"], + max_hours=config["electricity"]["max_hours"], input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", tech_costs=COSTS, @@ -414,9 +418,12 @@ rule add_extra_components: rule prepare_network: params: links=config["links"], - solving=config["solving"], lines=config["lines"], - electricity=config["electricity"], + solver_name=config["solving"]["solver"]["name"], + co2base=config["electricity"]["co2base"], + co2limit=config["electricity"]["co2limit"], + gaslimit=config["electricity"].get("gaslimit"), + max_hours=config["electricity"]["max_hours"], costs=config["costs"], input: RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 9faae4b9..da8a60f9 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -186,7 +186,7 @@ rule build_temperature_profiles: rule build_cop_profiles: params: - sector=config["sector"], + heat_pump_sink_T=config["sector"]["heat_pump_sink_T"], input: temp_soil_total=RESOURCES + "temp_soil_total_elec_s{simpl}_{clusters}.nc", temp_soil_rural=RESOURCES + "temp_soil_rural_elec_s{simpl}_{clusters}.nc", @@ -330,7 +330,7 @@ if config["sector"]["regional_co2_sequestration_potential"]["enable"]: rule build_sequestration_potentials: params: - sector=config["sector"], + co2seq_potential=config["sector"]["regional_co2_sequestration_potential"], input: sequestration_potential=HTTP.remote( "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson", @@ -405,8 +405,27 @@ rule build_ammonia_production: rule build_industry_sector_ratios: params: - industry=config["industry"], - sector=config["sector"], + industry={ + "H2_DRI":config["industry"]["H2_DRI"], + "elec_DRI":config["industry"]["elec_DRI"], + "HVC_production_today":config["industry"]["HVC_production_today"], + "petrochemical_process_emissions":config["industry"]["petrochemical_process_emissions"], + "NH3_process_emissions":config["industry"]["NH3_process_emissions"], + "MWh_CH4_per_tNH3_SMR":config["industry"]["MWh_CH4_per_tNH3_SMR"], + "MWh_elec_per_tNH3_SMR":config["industry"]["MWh_elec_per_tNH3_SMR"], + "chlorine_production_today":config["industry"]["chlorine_production_today"], + "MWh_H2_per_tCl":config["industry"]["MWh_H2_per_tCl"], + "MWh_elec_per_tCl":config["industry"]["MWh_elec_per_tCl"], + "methanol_production_today":config["industry"]["methanol_production_today"], + "MWh_CH4_per_tMeOH":config["industry"]["MWh_CH4_per_tMeOH"], + "MWh_elec_per_tMeOH":config["industry"]["MWh_elec_per_tMeOH"], + "MWh_elec_per_tHVC_mechanical_recycling":config["industry"]["MWh_elec_per_tHVC_mechanical_recycling"], + "MWh_elec_per_tHVC_chemical_recycling":config["industry"]["MWh_elec_per_tHVC_chemical_recycling"], + "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"], + "MWh_H2_per_tNH3_electrolysis":config["industry"]["MWh_H2_per_tNH3_electrolysis"], + "MWh_elec_per_tNH3_electrolysis":config["industry"]["MWh_elec_per_tNH3_electrolysis"] + }, + sector_amonia=config["sector"].get("ammonia", False), input: ammonia_production=RESOURCES + "ammonia_production.csv", idees="data/jrc-idees-2015", @@ -427,7 +446,12 @@ rule build_industry_sector_ratios: rule build_industrial_production_per_country: params: - industry=config["industry"], + industry={ + "reference_year":config["industry"]["reference_year"], + "HVC_production_today":config["industry"]["HVC_production_today"], + "chlorine_production_today":config["industry"]["chlorine_production_today"], + "methanol_production_today":config["industry"]["methanol_production_today"] + }, countries=config["countries"], input: ammonia_production=RESOURCES + "ammonia_production.csv", @@ -451,7 +475,14 @@ rule build_industrial_production_per_country: rule build_industrial_production_per_country_tomorrow: params: - industry=config["industry"], + industry={ + "St_primary_fraction":config["industry"]["St_primary_fraction"], + "DRI_fraction":config["industry"]["DRI_fraction"], + "Al_primary_fraction":config["industry"]["Al_primary_fraction"], + "HVC_mechanical_recycling_fraction":config["industry"]["HVC_mechanical_recycling_fraction"], + "HVC_chemical_recycling_fraction":config["industry"]["HVC_chemical_recycling_fraction"], + "HVC_primary_fraction":config["industry"]["HVC_primary_fraction"] + }, input: industrial_production_per_country=RESOURCES + "industrial_production_per_country.csv", @@ -477,7 +508,7 @@ rule build_industrial_production_per_country_tomorrow: rule build_industrial_distribution_key: params: - industry=config["industry"], + hotmaps_locate_missing=config["industry"].get("hotmaps_locate_missing", False), countries=config["countries"], input: regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -555,7 +586,12 @@ rule build_industrial_energy_demand_per_node: rule build_industrial_energy_demand_per_country_today: params: countries=config["countries"], - industry=config["industry"], + industry={ + "reference_year":config["industry"].get("reference_year", 2015), + "MWh_CH4_per_tNH3_SMR":config["industry"]["MWh_CH4_per_tNH3_SMR"], + "MWh_elec_per_tNH3_SMR":config["industry"]["MWh_elec_per_tNH3_SMR"], + "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"] + }, input: jrc="data/jrc-idees-2015", ammonia_production=RESOURCES + "ammonia_production.csv", @@ -603,7 +639,7 @@ if config["sector"]["retrofitting"]["retro_endogen"]: rule build_retro_cost: params: - sector=config["sector"], + retrofitting=config["sector"]["retrofitting"], countries=config["countries"], input: building_stock="data/retro/data_building_stock.csv", @@ -705,17 +741,23 @@ rule build_transport_demand: rule prepare_sector_network: params: co2_budget=config["co2_budget"], - solving=config["solving"], - existing_capacities=config["existing_capacities"], + solver_name=config["solving"]["solver"]["name"], + conventional_carriers=config["existing_capacities"]["conventional_carriers"], foresight=config["foresight"], costs=config["costs"], sector=config["sector"], - industry=config["industry"], + industry={ + "MWh_elec_per_tNH3_electrolysis":config["industry"]["MWh_elec_per_tNH3_electrolysis"], + "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"], + "MWh_H2_per_tNH3_electrolysis":config["industry"]["MWh_H2_per_tNH3_electrolysis"], + "MWh_NH3_per_MWh_H2_cracker":config["industry"]["MWh_NH3_per_MWh_H2_cracker"] + }, pypsa_eur=config["pypsa_eur"], - lines=config["lines"], - scenario=config["scenario"], + length_factor=config["lines"]["length_factor"], + planning_horizons=config["scenario"]["planning_horizons"], countries=config["countries"], - energy=config["energy"], + emissions_scope=config["energy"]["emissions"], + report_year=config["energy"]["eurostat_report_year"], RDIR=RDIR, input: **build_retro_cost_output, diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 9eb04ef5..ac80cd10 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -122,7 +122,8 @@ rule make_summary: rule plot_summary: params: countries=config["countries"], - scenario=config["scenario"], + planning_horizons=config["scenario"]["planning_horizons"], + sector_opts=config["scenario"]["sector_opts"], plotting=config["plotting"], RDIR=RDIR, input: diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 30520dc1..892415c4 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -41,7 +41,7 @@ rule solve_network: rule solve_operations_network: params: - solving=config["solving"], + options=config["solving"]["options"], input: network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 22555f75..8ec1c4a8 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -5,7 +5,7 @@ rule add_existing_baseyear: params: - scenario=config["scenario"], + baseyear=config["scenario"]["planning_horizons"][0], sector=config["sector"], existing_capacities=config["existing_capacities"], costs=config["costs"], @@ -48,8 +48,9 @@ rule add_existing_baseyear: rule add_brownfield: params: - sector=config["sector"], - existing_capacities=config["existing_capacities"], + H2_retrofit=config["sector"]["H2_retrofit"], + H2_retrofit_capacity_per_CH4=config["sector"]["H2_retrofit_capacity_per_CH4"], + threshold_capacity=config["existing_capacities"]["threshold_capacity"], input: overrides="data/override_component_attrs", network=RESULTS diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 9330953b..48dffbb9 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -49,7 +49,7 @@ def add_brownfield(n, n_p, year): ) ] - threshold = snakemake.params["existing_capacities"]["threshold_capacity"] + threshold = snakemake.params["threshold_capacity"] if not chp_heat.empty: threshold_chp_heat = ( @@ -87,7 +87,7 @@ def add_brownfield(n, n_p, year): # deal with gas network pipe_carrier = ["gas pipeline"] - if snakemake.params["sector"]["H2_retrofit"]: + if snakemake.params["H2_retrofit"]: # drop capacities of previous year to avoid duplicating to_drop = n.links.carrier.isin(pipe_carrier) & (n.links.build_year != year) n.mremove("Link", n.links.loc[to_drop].index) @@ -98,7 +98,7 @@ def add_brownfield(n, n_p, year): & (n.links.build_year != year) ].index gas_pipes_i = n.links[n.links.carrier.isin(pipe_carrier)].index - CH4_per_H2 = 1 / snakemake.params["sector"]["H2_retrofit_capacity_per_CH4"] + CH4_per_H2 = 1 / snakemake.params["H2_retrofit_capacity_per_CH4"] fr = "H2 pipeline retrofitted" to = "gas pipeline" # today's pipe capacity diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 69d91b87..dc5fb7cc 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -137,7 +137,7 @@ def _add_missing_carriers_from_costs(n, costs, carriers): n.import_components_from_dataframe(emissions, "Carrier") -def load_costs(tech_costs, params, elec_params, Nyears=1.0): +def load_costs(tech_costs, params, max_hours, Nyears=1.0): # set all asset costs and other parameters costs = pd.read_csv(tech_costs, index_col=[0, 1]).sort_index() @@ -180,7 +180,6 @@ def load_costs(tech_costs, params, elec_params, Nyears=1.0): dict(capital_cost=capital_cost, marginal_cost=0.0, co2_emissions=0.0) ) - max_hours = elec_params["max_hours"] costs.loc["battery"] = costs_for_storage( costs.loc["battery storage"], costs.loc["battery inverter"], @@ -728,7 +727,7 @@ if __name__ == "__main__": costs = load_costs( snakemake.input.tech_costs, snakemake.params["costs"], - snakemake.params["electricity"], + snakemake.params["electricity"]["max_hours"], Nyears, ) ppl = load_powerplants(snakemake.input.powerplants) @@ -759,10 +758,10 @@ if __name__ == "__main__": snakemake.input.load, snakemake.input.nuts3_shapes, snakemake.params["countries"], - snakemake.params["load"]["scaling_factor"], + snakemake.params["scaling_factor"], ) - update_transmission_costs(n, costs, snakemake.params["lines"]["length_factor"]) + update_transmission_costs(n, costs, snakemake.params["length_factor"]) conventional_inputs = { k: v for k, v in snakemake.input.items() if k.startswith("conventional_") @@ -783,7 +782,7 @@ if __name__ == "__main__": snakemake.input, renewable_carriers, extendable_carriers, - snakemake.params["lines"]["length_factor"], + snakemake.params["length_factor"], ) if "hydro" in renewable_carriers: diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index a4d3748b..bc1d3a05 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -615,7 +615,7 @@ if __name__ == "__main__": options = snakemake.params["sector"] opts = snakemake.wildcards.sector_opts.split("-") - baseyear = snakemake.params["scenario"]["planning_horizons"][0] + baseyear = snakemake.params["baseyear"] overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index 08178c0a..ce4e533e 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -67,9 +67,8 @@ idx = pd.IndexSlice logger = logging.getLogger(__name__) -def attach_storageunits(n, costs, elec_opts): - carriers = elec_opts["extendable_carriers"]["StorageUnit"] - max_hours = elec_opts["max_hours"] +def attach_storageunits(n, costs, ext_carriers, max_hours): + carriers = ext_carriers["StorageUnit"] _add_missing_carriers_from_costs(n, costs, carriers) @@ -99,8 +98,8 @@ def attach_storageunits(n, costs, elec_opts): ) -def attach_stores(n, costs, elec_opts): - carriers = elec_opts["extendable_carriers"]["Store"] +def attach_stores(n, costs, ext_carriers): + carriers = ext_carriers["Store"] _add_missing_carriers_from_costs(n, costs, carriers) @@ -187,8 +186,7 @@ def attach_stores(n, costs, elec_opts): ) -def attach_hydrogen_pipelines(n, costs, elec_opts): - ext_carriers = elec_opts["extendable_carriers"] +def attach_hydrogen_pipelines(n, costs, ext_carriers): as_stores = ext_carriers.get("Store", []) if "H2 pipeline" not in ext_carriers.get("Link", []): @@ -235,16 +233,17 @@ if __name__ == "__main__": configure_logging(snakemake) n = pypsa.Network(snakemake.input.network) - elec_config = snakemake.params["electricity"] + ext_carriers = snakemake.params["ext_carriers"] + max_hours = snakemake.params["max_hours"] Nyears = n.snapshot_weightings.objective.sum() / 8760.0 costs = load_costs( - snakemake.input.tech_costs, snakemake.params["costs"], elec_config, Nyears + snakemake.input.tech_costs, snakemake.params["costs"], max_hours, Nyears ) - attach_storageunits(n, costs, elec_config) - attach_stores(n, costs, elec_config) - attach_hydrogen_pipelines(n, costs, elec_config) + attach_storageunits(n, costs, ext_carriers, max_hours) + attach_stores(n, costs, ext_carriers) + attach_hydrogen_pipelines(n, costs, ext_carriers) add_nice_carrier_names(n, snakemake.config) diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py index 7128ec0d..983eda2d 100644 --- a/scripts/build_cop_profiles.py +++ b/scripts/build_cop_profiles.py @@ -39,7 +39,7 @@ if __name__ == "__main__": for source in ["air", "soil"]: source_T = xr.open_dataarray(snakemake.input[f"temp_{source}_{area}"]) - delta_T = snakemake.params["sector"]["heat_pump_sink_T"] - source_T + delta_T = snakemake.params["heat_pump_sink_T"] - source_T cop = coefficient_of_performance(delta_T, source) diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 2f61f017..798588d9 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -106,7 +106,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_cutout", cutout="europe-2013-era5") configure_logging(snakemake) - cutout_params = snakemake.params["atlite"]["cutouts"][snakemake.wildcards.cutout] + cutout_params = snakemake.params["cutouts"][snakemake.wildcards.cutout] snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"]) time = [snapshots[0], snapshots[-1]] diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index dd686be3..26bf31c6 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -130,7 +130,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_hydro_profile") configure_logging(snakemake) - params_hydro = snakemake.params["renewable"]["hydro"] + params_hydro = snakemake.params["hydro"] cutout = atlite.Cutout(snakemake.input.cutout) countries = snakemake.params["countries"] diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index 9a513673..a51ff8b2 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -73,7 +73,7 @@ def prepare_hotmaps_database(regions): df[["srid", "coordinates"]] = df.geom.str.split(";", expand=True) - if snakemake.params["industry"].get("hotmaps_locate_missing", False): + if snakemake.params["hotmaps_locate_missing"]: df = locate_missing_industrial_sites(df) # remove those sites without valid locations diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index 2ec007a9..fa9e5c18 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -439,7 +439,7 @@ def chemicals_industry(): sector = "Ammonia" df[sector] = 0.0 - if snakemake.params["sector"].get("ammonia", False): + if snakemake.params["sector_amonia"]: df.loc["ammonia", sector] = params["MWh_NH3_per_tNH3"] else: df.loc["hydrogen", sector] = params["MWh_H2_per_tNH3_electrolysis"] diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index bd0ee74e..6edd4ac4 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -134,12 +134,12 @@ if __name__ == "__main__": ppl = ppl.query('not (Country in @available_countries and Fueltype == "Bioenergy")') ppl = pd.concat([ppl, opsd]) - ppl_query = snakemake.params["electricity"]["powerplants_filter"] + ppl_query = snakemake.params["powerplants_filter"] if isinstance(ppl_query, str): ppl.query(ppl_query, inplace=True) # add carriers from own powerplant files: - custom_ppl_query = snakemake.params["electricity"]["custom_powerplants"] + custom_ppl_query = snakemake.params["custom_powerplants"] ppl = add_custom_powerplants( ppl, snakemake.input.custom_powerplants, custom_ppl_query ) diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index c52d4eb6..8473be7a 100644 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -1040,7 +1040,7 @@ if __name__ == "__main__": # ******** config ********************************************************* - retro_opts = snakemake.params["sector"]["retrofitting"] + retro_opts = snakemake.params["retrofitting"] interest_rate = retro_opts["interest_rate"] annualise_cost = retro_opts["annualise_cost"] # annualise the investment costs tax_weighting = retro_opts[ diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py index 5c388b2e..9d26b0b9 100644 --- a/scripts/build_sequestration_potentials.py +++ b/scripts/build_sequestration_potentials.py @@ -41,7 +41,7 @@ if __name__ == "__main__": "build_sequestration_potentials", simpl="", clusters="181" ) - cf = snakemake.params["sector"]["regional_co2_sequestration_potential"] + cf = snakemake.params["co2seq_potential"] gdf = gpd.read_file(snakemake.input.sequestration_potential[0]) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index d217e5ed..3cfbd214 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -479,7 +479,7 @@ if __name__ == "__main__": if snakemake.wildcards.clusters.endswith("m"): n_clusters = int(snakemake.wildcards.clusters[:-1]) conventional = set( - snakemake.params["electricity"].get("conventional_carriers", []) + snakemake.params["conventional_carriers"] ) aggregate_carriers = conventional.intersection(aggregate_carriers) elif snakemake.wildcards.clusters == "all": @@ -495,13 +495,13 @@ if __name__ == "__main__": n, busmap, linemap, linemap, pd.Series(dtype="O") ) else: - line_length_factor = snakemake.params["lines"]["length_factor"] + line_length_factor = snakemake.params["length_factor"] Nyears = n.snapshot_weightings.objective.sum() / 8760 hvac_overhead_cost = load_costs( snakemake.input.tech_costs, snakemake.params["costs"], - snakemake.params["electricity"], + snakemake.params["max_hours"], Nyears, ).at["HVAC overhead", "capital_cost"] @@ -539,7 +539,7 @@ if __name__ == "__main__": aggregate_carriers, line_length_factor, aggregation_strategies, - snakemake.params["solving"]["solver"]["name"], + snakemake.params["solver_name"], cluster_config.get("algorithm", "hac"), cluster_config.get("feature", "solar+onwind-time"), hvac_overhead_cost, diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 4aa37de5..36f75207 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -455,7 +455,7 @@ def plot_carbon_budget_distribution(input_eurostat): ax1 = plt.subplot(gs1[0, 0]) ax1.set_ylabel("CO$_2$ emissions (Gt per year)", fontsize=22) ax1.set_ylim([0, 5]) - ax1.set_xlim([1990, snakemake.params["scenario"]["planning_horizons"][-1] + 1]) + ax1.set_xlim([1990, snakemake.params["planning_horizons"][-1] + 1]) path_cb = "results/" + snakemake.params.RDIR + "/csvs/" countries = snakemake.params["countries"] @@ -555,7 +555,7 @@ if __name__ == "__main__": plot_balances() - for sector_opts in snakemake.params["scenario"]["sector_opts"]: + for sector_opts in snakemake.params["sector_opts"]: opts = sector_opts.split("-") for o in opts: if "cb" in o: diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 46e4e74a..51777ef5 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -254,7 +254,7 @@ if __name__ == "__main__": costs = load_costs( snakemake.input.tech_costs, snakemake.params["costs"], - snakemake.params["electricity"], + snakemake.params["max_hours"], Nyears, ) @@ -269,7 +269,7 @@ if __name__ == "__main__": for o in opts: m = re.match(r"^\d+seg$", o, re.IGNORECASE) if m is not None: - solver_name = snakemake.params["solving"]["solver"]["name"] + solver_name = snakemake.params["solver_name"] n = apply_time_segmentation(n, m.group(0)[:-3], solver_name) break @@ -277,11 +277,11 @@ if __name__ == "__main__": if "Co2L" in o: m = re.findall("[0-9]*\.?[0-9]+$", o) if len(m) > 0: - co2limit = float(m[0]) * snakemake.params["electricity"]["co2base"] + co2limit = float(m[0]) * snakemake.params["co2base"] add_co2limit(n, co2limit, Nyears) logger.info("Setting CO2 limit according to wildcard value.") else: - add_co2limit(n, snakemake.params["electricity"]["co2limit"], Nyears) + add_co2limit(n, snakemake.params["co2limit"], Nyears) logger.info("Setting CO2 limit according to config value.") break @@ -293,7 +293,7 @@ if __name__ == "__main__": add_gaslimit(n, limit, Nyears) logger.info("Setting gas usage limit according to wildcard value.") else: - add_gaslimit(n, snakemake.params["electricity"].get("gaslimit"), Nyears) + add_gaslimit(n, snakemake.params["gaslimit"], Nyears) logger.info("Setting gas usage limit according to config value.") break diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 6a71b1e2..54ab1f94 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -252,7 +252,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): countries, input_eurostat, opts, emissions_scope, report_year, year=2018 ) - planning_horizons = snakemake.params["scenario"]["planning_horizons"] + planning_horizons = snakemake.params["planning_horizons"] t_0 = planning_horizons[0] if "be" in o: @@ -391,7 +391,7 @@ def update_wind_solar_costs(n, costs): with xr.open_dataset(profile) as ds: underwater_fraction = ds["underwater_fraction"].to_pandas() connection_cost = ( - snakemake.params["lines"]["length_factor"] + snakemake.params["length_factor"] * ds["average_distance"].to_pandas() * ( underwater_fraction @@ -3300,7 +3300,7 @@ if __name__ == "__main__": if snakemake.params["foresight"] == "myopic": add_lifetime_wind_solar(n, costs) - conventional = snakemake.params["existing_capacities"]["conventional_carriers"] + conventional = snakemake.params["conventional_carriers"] for carrier in conventional: add_carrier_buses(n, carrier) @@ -3365,7 +3365,7 @@ if __name__ == "__main__": if options["allam_cycle"]: add_allam(n, costs) - solver_name = snakemake.params["solving"]["solver"]["name"] + solver_name = snakemake.params["solver_name"] n = set_temporal_aggregation(n, opts, solver_name) limit_type = "config" @@ -3376,8 +3376,8 @@ if __name__ == "__main__": limit_type = "carbon budget" fn = "results/" + snakemake.params.RDIR + "/csvs/carbon_budget_distribution.csv" if not os.path.exists(fn): - emissions_scope = snakemake.params["energy"]["emissions"] - report_year = snakemake.params["energy"]["eurostat_report_year"] + emissions_scope = snakemake.params["emissions_scope"] + report_year = snakemake.params["report_year"] build_carbon_budget( o, snakemake.input.eurostat, fn, emissions_scope, report_year ) @@ -3413,7 +3413,7 @@ if __name__ == "__main__": add_electricity_grid_connection(n, costs) first_year_myopic = (snakemake.params["foresight"] == "myopic") and ( - snakemake.params["scenario"]["planning_horizons"][0] == investment_year + snakemake.params["planning_horizons"][0] == investment_year ) if options.get("cluster_heat_buses", False) and not first_year_myopic: diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 689e8084..83e932a3 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -560,7 +560,7 @@ if __name__ == "__main__": technology_costs = load_costs( snakemake.input.tech_costs, snakemake.params["costs"], - snakemake.params["electricity"], + snakemake.params["max_hours"], Nyears, ) diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index c268e2ee..421bc515 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -41,7 +41,7 @@ if __name__ == "__main__": opts = (snakemake.wildcards.opts + "-" + snakemake.wildcards.sector_opts).split("-") opts = [o for o in opts if o != ""] - solve_opts = snakemake.params["solving"]["options"] + solve_opts = snakemake.params["options"] np.random.seed(solve_opts.get("seed", 123)) From f3e8fe23122bb47e2d8dfb4c8264e920007ee20b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 27 May 2023 13:50:58 +0000 Subject: [PATCH 28/41] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_sector.smk | 100 ++++++++++++++++++++++++------------- scripts/cluster_network.py | 4 +- 2 files changed, 65 insertions(+), 39 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index da8a60f9..2d6c8263 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -406,24 +406,38 @@ rule build_ammonia_production: rule build_industry_sector_ratios: params: industry={ - "H2_DRI":config["industry"]["H2_DRI"], - "elec_DRI":config["industry"]["elec_DRI"], - "HVC_production_today":config["industry"]["HVC_production_today"], - "petrochemical_process_emissions":config["industry"]["petrochemical_process_emissions"], - "NH3_process_emissions":config["industry"]["NH3_process_emissions"], - "MWh_CH4_per_tNH3_SMR":config["industry"]["MWh_CH4_per_tNH3_SMR"], - "MWh_elec_per_tNH3_SMR":config["industry"]["MWh_elec_per_tNH3_SMR"], - "chlorine_production_today":config["industry"]["chlorine_production_today"], - "MWh_H2_per_tCl":config["industry"]["MWh_H2_per_tCl"], - "MWh_elec_per_tCl":config["industry"]["MWh_elec_per_tCl"], - "methanol_production_today":config["industry"]["methanol_production_today"], - "MWh_CH4_per_tMeOH":config["industry"]["MWh_CH4_per_tMeOH"], - "MWh_elec_per_tMeOH":config["industry"]["MWh_elec_per_tMeOH"], - "MWh_elec_per_tHVC_mechanical_recycling":config["industry"]["MWh_elec_per_tHVC_mechanical_recycling"], - "MWh_elec_per_tHVC_chemical_recycling":config["industry"]["MWh_elec_per_tHVC_chemical_recycling"], - "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"], - "MWh_H2_per_tNH3_electrolysis":config["industry"]["MWh_H2_per_tNH3_electrolysis"], - "MWh_elec_per_tNH3_electrolysis":config["industry"]["MWh_elec_per_tNH3_electrolysis"] + "H2_DRI": config["industry"]["H2_DRI"], + "elec_DRI": config["industry"]["elec_DRI"], + "HVC_production_today": config["industry"]["HVC_production_today"], + "petrochemical_process_emissions": config["industry"][ + "petrochemical_process_emissions" + ], + "NH3_process_emissions": config["industry"]["NH3_process_emissions"], + "MWh_CH4_per_tNH3_SMR": config["industry"]["MWh_CH4_per_tNH3_SMR"], + "MWh_elec_per_tNH3_SMR": config["industry"]["MWh_elec_per_tNH3_SMR"], + "chlorine_production_today": config["industry"][ + "chlorine_production_today" + ], + "MWh_H2_per_tCl": config["industry"]["MWh_H2_per_tCl"], + "MWh_elec_per_tCl": config["industry"]["MWh_elec_per_tCl"], + "methanol_production_today": config["industry"][ + "methanol_production_today" + ], + "MWh_CH4_per_tMeOH": config["industry"]["MWh_CH4_per_tMeOH"], + "MWh_elec_per_tMeOH": config["industry"]["MWh_elec_per_tMeOH"], + "MWh_elec_per_tHVC_mechanical_recycling": config["industry"][ + "MWh_elec_per_tHVC_mechanical_recycling" + ], + "MWh_elec_per_tHVC_chemical_recycling": config["industry"][ + "MWh_elec_per_tHVC_chemical_recycling" + ], + "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"], + "MWh_H2_per_tNH3_electrolysis": config["industry"][ + "MWh_H2_per_tNH3_electrolysis" + ], + "MWh_elec_per_tNH3_electrolysis": config["industry"][ + "MWh_elec_per_tNH3_electrolysis" + ], }, sector_amonia=config["sector"].get("ammonia", False), input: @@ -447,10 +461,14 @@ rule build_industry_sector_ratios: rule build_industrial_production_per_country: params: industry={ - "reference_year":config["industry"]["reference_year"], - "HVC_production_today":config["industry"]["HVC_production_today"], - "chlorine_production_today":config["industry"]["chlorine_production_today"], - "methanol_production_today":config["industry"]["methanol_production_today"] + "reference_year": config["industry"]["reference_year"], + "HVC_production_today": config["industry"]["HVC_production_today"], + "chlorine_production_today": config["industry"][ + "chlorine_production_today" + ], + "methanol_production_today": config["industry"][ + "methanol_production_today" + ], }, countries=config["countries"], input: @@ -476,12 +494,16 @@ rule build_industrial_production_per_country: rule build_industrial_production_per_country_tomorrow: params: industry={ - "St_primary_fraction":config["industry"]["St_primary_fraction"], - "DRI_fraction":config["industry"]["DRI_fraction"], - "Al_primary_fraction":config["industry"]["Al_primary_fraction"], - "HVC_mechanical_recycling_fraction":config["industry"]["HVC_mechanical_recycling_fraction"], - "HVC_chemical_recycling_fraction":config["industry"]["HVC_chemical_recycling_fraction"], - "HVC_primary_fraction":config["industry"]["HVC_primary_fraction"] + "St_primary_fraction": config["industry"]["St_primary_fraction"], + "DRI_fraction": config["industry"]["DRI_fraction"], + "Al_primary_fraction": config["industry"]["Al_primary_fraction"], + "HVC_mechanical_recycling_fraction": config["industry"][ + "HVC_mechanical_recycling_fraction" + ], + "HVC_chemical_recycling_fraction": config["industry"][ + "HVC_chemical_recycling_fraction" + ], + "HVC_primary_fraction": config["industry"]["HVC_primary_fraction"], }, input: industrial_production_per_country=RESOURCES @@ -587,10 +609,10 @@ rule build_industrial_energy_demand_per_country_today: params: countries=config["countries"], industry={ - "reference_year":config["industry"].get("reference_year", 2015), - "MWh_CH4_per_tNH3_SMR":config["industry"]["MWh_CH4_per_tNH3_SMR"], - "MWh_elec_per_tNH3_SMR":config["industry"]["MWh_elec_per_tNH3_SMR"], - "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"] + "reference_year": config["industry"].get("reference_year", 2015), + "MWh_CH4_per_tNH3_SMR": config["industry"]["MWh_CH4_per_tNH3_SMR"], + "MWh_elec_per_tNH3_SMR": config["industry"]["MWh_elec_per_tNH3_SMR"], + "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"], }, input: jrc="data/jrc-idees-2015", @@ -747,10 +769,16 @@ rule prepare_sector_network: costs=config["costs"], sector=config["sector"], industry={ - "MWh_elec_per_tNH3_electrolysis":config["industry"]["MWh_elec_per_tNH3_electrolysis"], - "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"], - "MWh_H2_per_tNH3_electrolysis":config["industry"]["MWh_H2_per_tNH3_electrolysis"], - "MWh_NH3_per_MWh_H2_cracker":config["industry"]["MWh_NH3_per_MWh_H2_cracker"] + "MWh_elec_per_tNH3_electrolysis": config["industry"][ + "MWh_elec_per_tNH3_electrolysis" + ], + "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"], + "MWh_H2_per_tNH3_electrolysis": config["industry"][ + "MWh_H2_per_tNH3_electrolysis" + ], + "MWh_NH3_per_MWh_H2_cracker": config["industry"][ + "MWh_NH3_per_MWh_H2_cracker" + ], }, pypsa_eur=config["pypsa_eur"], length_factor=config["lines"]["length_factor"], diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 3cfbd214..b86bb225 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -478,9 +478,7 @@ if __name__ == "__main__": aggregate_carriers = set(n.generators.carrier) - set(exclude_carriers) if snakemake.wildcards.clusters.endswith("m"): n_clusters = int(snakemake.wildcards.clusters[:-1]) - conventional = set( - snakemake.params["conventional_carriers"] - ) + conventional = set(snakemake.params["conventional_carriers"]) aggregate_carriers = conventional.intersection(aggregate_carriers) elif snakemake.wildcards.clusters == "all": n_clusters = len(n.buses) From b67a00903af6b5d55019da50bd845a217e07d628 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Wed, 31 May 2023 10:54:27 +0200 Subject: [PATCH 29/41] using higher keys for industries --- rules/build_electricity.smk | 1 - rules/build_sector.smk | 79 +++---------------------------------- 2 files changed, 5 insertions(+), 75 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 9b3a03fb..34276ba4 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -255,7 +255,6 @@ rule build_hydro_profile: params: hydro=config["renewable"]["hydro"], countries=config["countries"], - renewable=config["renewable"], input: country_shapes=RESOURCES + "country_shapes.geojson", eia_hydro_generation="data/eia_hydro_annual_generation.csv", diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 2d6c8263..3714cbad 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -405,40 +405,7 @@ rule build_ammonia_production: rule build_industry_sector_ratios: params: - industry={ - "H2_DRI": config["industry"]["H2_DRI"], - "elec_DRI": config["industry"]["elec_DRI"], - "HVC_production_today": config["industry"]["HVC_production_today"], - "petrochemical_process_emissions": config["industry"][ - "petrochemical_process_emissions" - ], - "NH3_process_emissions": config["industry"]["NH3_process_emissions"], - "MWh_CH4_per_tNH3_SMR": config["industry"]["MWh_CH4_per_tNH3_SMR"], - "MWh_elec_per_tNH3_SMR": config["industry"]["MWh_elec_per_tNH3_SMR"], - "chlorine_production_today": config["industry"][ - "chlorine_production_today" - ], - "MWh_H2_per_tCl": config["industry"]["MWh_H2_per_tCl"], - "MWh_elec_per_tCl": config["industry"]["MWh_elec_per_tCl"], - "methanol_production_today": config["industry"][ - "methanol_production_today" - ], - "MWh_CH4_per_tMeOH": config["industry"]["MWh_CH4_per_tMeOH"], - "MWh_elec_per_tMeOH": config["industry"]["MWh_elec_per_tMeOH"], - "MWh_elec_per_tHVC_mechanical_recycling": config["industry"][ - "MWh_elec_per_tHVC_mechanical_recycling" - ], - "MWh_elec_per_tHVC_chemical_recycling": config["industry"][ - "MWh_elec_per_tHVC_chemical_recycling" - ], - "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"], - "MWh_H2_per_tNH3_electrolysis": config["industry"][ - "MWh_H2_per_tNH3_electrolysis" - ], - "MWh_elec_per_tNH3_electrolysis": config["industry"][ - "MWh_elec_per_tNH3_electrolysis" - ], - }, + industry=config["industry"], sector_amonia=config["sector"].get("ammonia", False), input: ammonia_production=RESOURCES + "ammonia_production.csv", @@ -460,16 +427,7 @@ rule build_industry_sector_ratios: rule build_industrial_production_per_country: params: - industry={ - "reference_year": config["industry"]["reference_year"], - "HVC_production_today": config["industry"]["HVC_production_today"], - "chlorine_production_today": config["industry"][ - "chlorine_production_today" - ], - "methanol_production_today": config["industry"][ - "methanol_production_today" - ], - }, + industry=config["industry"], countries=config["countries"], input: ammonia_production=RESOURCES + "ammonia_production.csv", @@ -493,18 +451,7 @@ rule build_industrial_production_per_country: rule build_industrial_production_per_country_tomorrow: params: - industry={ - "St_primary_fraction": config["industry"]["St_primary_fraction"], - "DRI_fraction": config["industry"]["DRI_fraction"], - "Al_primary_fraction": config["industry"]["Al_primary_fraction"], - "HVC_mechanical_recycling_fraction": config["industry"][ - "HVC_mechanical_recycling_fraction" - ], - "HVC_chemical_recycling_fraction": config["industry"][ - "HVC_chemical_recycling_fraction" - ], - "HVC_primary_fraction": config["industry"]["HVC_primary_fraction"], - }, + industry=config["industry"], input: industrial_production_per_country=RESOURCES + "industrial_production_per_country.csv", @@ -608,12 +555,7 @@ rule build_industrial_energy_demand_per_node: rule build_industrial_energy_demand_per_country_today: params: countries=config["countries"], - industry={ - "reference_year": config["industry"].get("reference_year", 2015), - "MWh_CH4_per_tNH3_SMR": config["industry"]["MWh_CH4_per_tNH3_SMR"], - "MWh_elec_per_tNH3_SMR": config["industry"]["MWh_elec_per_tNH3_SMR"], - "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"], - }, + industry=config["industry"], input: jrc="data/jrc-idees-2015", ammonia_production=RESOURCES + "ammonia_production.csv", @@ -768,18 +710,7 @@ rule prepare_sector_network: foresight=config["foresight"], costs=config["costs"], sector=config["sector"], - industry={ - "MWh_elec_per_tNH3_electrolysis": config["industry"][ - "MWh_elec_per_tNH3_electrolysis" - ], - "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"], - "MWh_H2_per_tNH3_electrolysis": config["industry"][ - "MWh_H2_per_tNH3_electrolysis" - ], - "MWh_NH3_per_MWh_H2_cracker": config["industry"][ - "MWh_NH3_per_MWh_H2_cracker" - ], - }, + industry=config["industry"], pypsa_eur=config["pypsa_eur"], length_factor=config["lines"]["length_factor"], planning_horizons=config["scenario"]["planning_horizons"], From 83ff639ad57f40f7a61027951843cd7e4b334642 Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Fri, 2 Jun 2023 12:52:49 +0200 Subject: [PATCH 30/41] amend solve_network --- rules/solve_electricity.smk | 13 +++---------- rules/solve_myopic.smk | 13 +++---------- rules/solve_overnight.smk | 13 +++---------- scripts/solve_network.py | 25 +++++++++++++++---------- 4 files changed, 24 insertions(+), 40 deletions(-) diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 892415c4..e7b264f7 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -6,16 +6,9 @@ rule solve_network: params: solving=config["solving"], - config_parts={ - "foresight": config["foresight"], - "planning_horizons": config["scenario"]["planning_horizons"], - "co2_sequestration_potential": config["sector"].get( - "co2_sequestration_potential", 200 - ), - "H2_retrofit_capacity_per_CH4": config["sector"][ - "H2_retrofit_capacity_per_CH4" - ], - }, + foresight=config["foresight"], + planning_horizons=config["scenario"]["planning_horizons"], + co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200), input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 8ec1c4a8..39567b92 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -85,16 +85,9 @@ ruleorder: add_existing_baseyear > add_brownfield rule solve_sector_network_myopic: params: solving=config["solving"], - config_parts={ - "foresight": config["foresight"], - "planning_horizons": config["scenario"]["planning_horizons"], - "co2_sequestration_potential": config["sector"].get( - "co2_sequestration_potential", 200 - ), - "H2_retrofit_capacity_per_CH4": config["sector"][ - "H2_retrofit_capacity_per_CH4" - ], - }, + foresight=config["foresight"], + planning_horizons=config["scenario"]["planning_horizons"], + co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200), input: overrides="data/override_component_attrs", network=RESULTS diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index a523f132..71ef075e 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -6,16 +6,9 @@ rule solve_sector_network: params: solving=config["solving"], - config_parts={ - "foresight": config["foresight"], - "planning_horizons": config["scenario"]["planning_horizons"], - "co2_sequestration_potential": config["sector"].get( - "co2_sequestration_potential", 200 - ), - "H2_retrofit_capacity_per_CH4": config["sector"][ - "H2_retrofit_capacity_per_CH4" - ], - }, + foresight=config["foresight"], + planning_horizons=config["scenario"]["planning_horizons"], + co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200), input: overrides="data/override_component_attrs", network=RESULTS diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 6e07e340..90ff3a2e 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -44,14 +44,14 @@ pypsa.pf.logger.setLevel(logging.WARNING) from pypsa.descriptors import get_switchable_as_dense as get_as_dense -def add_land_use_constraint(n, param, config): +def add_land_use_constraint(n, planning_horizons_param, config): if "m" in snakemake.wildcards.clusters: - _add_land_use_constraint_m(n, param, config) + _add_land_use_constraint_m(n, planning_horizons_param, config) else: - _add_land_use_constraint(n, param) + _add_land_use_constraint(n) -def _add_land_use_constraint(n, param): +def _add_land_use_constraint(n): # warning: this will miss existing offwind which is not classed AC-DC and has carrier 'offwind' for carrier in ["solar", "onwind", "offwind-ac", "offwind-dc"]: @@ -80,7 +80,7 @@ def _add_land_use_constraint(n, param): n.generators.p_nom_max.clip(lower=0, inplace=True) -def _add_land_use_constraint_m(n, param, config): +def _add_land_use_constraint_m(n, planning_horizons_param, config): # if generators clustering is lower than network clustering, land_use accounting is at generators clusters planning_horizons = param["planning_horizons"] @@ -141,7 +141,7 @@ def add_co2_sequestration_limit(n, limit=200): ) -def prepare_network(n, solve_opts=None, config=None, param=None): +def prepare_network(n, solve_opts=None, config=None, foresight_param=None, planning_horizons_param=None, co2_sequestration_potential=None): if "clip_p_max_pu" in solve_opts: for df in ( n.generators_t.p_max_pu, @@ -191,11 +191,11 @@ def prepare_network(n, solve_opts=None, config=None, param=None): n.set_snapshots(n.snapshots[:nhours]) n.snapshot_weightings[:] = 8760.0 / nhours - if param["foresight"] == "myopic": - add_land_use_constraint(n, param, config) + if foresight_param == "myopic": + add_land_use_constraint(n, planning_horizons_param, config) if n.stores.carrier.eq("co2 stored").any(): - limit = param["co2_sequestration_potential"] + limit = co2_sequestration_potential_param add_co2_sequestration_limit(n, limit=limit) return n @@ -676,7 +676,12 @@ if __name__ == "__main__": n = pypsa.Network(snakemake.input.network) n = prepare_network( - n, solve_opts, config=snakemake.config, param=snakemake.params["config_parts"] + n, + solve_opts, + config=snakemake.config, + foresight_param=snakemake.params["foresight"], + planning_horizons_param=snakemake.params["planning_horizons"], + co2_sequestration_potential=snakemake.params["co2_sequestration_potential"] ) n = solve_network( From 0f5bb8b893afc4955f608d9a206035f63e2bb6fe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 2 Jun 2023 10:53:09 +0000 Subject: [PATCH 31/41] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/solve_electricity.smk | 4 +++- rules/solve_myopic.smk | 4 +++- rules/solve_overnight.smk | 4 +++- scripts/solve_network.py | 11 +++++++++-- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index e7b264f7..f73a5b0c 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -8,7 +8,9 @@ rule solve_network: solving=config["solving"], foresight=config["foresight"], planning_horizons=config["scenario"]["planning_horizons"], - co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200), + co2_sequestration_potential=config["sector"].get( + "co2_sequestration_potential", 200 + ), input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 39567b92..086375c2 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -87,7 +87,9 @@ rule solve_sector_network_myopic: solving=config["solving"], foresight=config["foresight"], planning_horizons=config["scenario"]["planning_horizons"], - co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200), + co2_sequestration_potential=config["sector"].get( + "co2_sequestration_potential", 200 + ), input: overrides="data/override_component_attrs", network=RESULTS diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index 71ef075e..fc430f4d 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -8,7 +8,9 @@ rule solve_sector_network: solving=config["solving"], foresight=config["foresight"], planning_horizons=config["scenario"]["planning_horizons"], - co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200), + co2_sequestration_potential=config["sector"].get( + "co2_sequestration_potential", 200 + ), input: overrides="data/override_component_attrs", network=RESULTS diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 90ff3a2e..ab1175ab 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -141,7 +141,14 @@ def add_co2_sequestration_limit(n, limit=200): ) -def prepare_network(n, solve_opts=None, config=None, foresight_param=None, planning_horizons_param=None, co2_sequestration_potential=None): +def prepare_network( + n, + solve_opts=None, + config=None, + foresight_param=None, + planning_horizons_param=None, + co2_sequestration_potential=None, +): if "clip_p_max_pu" in solve_opts: for df in ( n.generators_t.p_max_pu, @@ -681,7 +688,7 @@ if __name__ == "__main__": config=snakemake.config, foresight_param=snakemake.params["foresight"], planning_horizons_param=snakemake.params["planning_horizons"], - co2_sequestration_potential=snakemake.params["co2_sequestration_potential"] + co2_sequestration_potential=snakemake.params["co2_sequestration_potential"], ) n = solve_network( From 8dbcca560cd47d58d098dd91e751efd45314b8e7 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Sat, 3 Jun 2023 12:09:51 +0200 Subject: [PATCH 32/41] minor fixes in solve_network.py --- scripts/solve_network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index ab1175ab..ea5446ad 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -147,7 +147,7 @@ def prepare_network( config=None, foresight_param=None, planning_horizons_param=None, - co2_sequestration_potential=None, + co2_sequestration_potential_param=None, ): if "clip_p_max_pu" in solve_opts: for df in ( @@ -688,7 +688,7 @@ if __name__ == "__main__": config=snakemake.config, foresight_param=snakemake.params["foresight"], planning_horizons_param=snakemake.params["planning_horizons"], - co2_sequestration_potential=snakemake.params["co2_sequestration_potential"], + co2_sequestration_potential_param=snakemake.params["co2_sequestration_potential"], ) n = solve_network( From d6057cb92be6fdb07d02da23e13035b90a57d1fc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 3 Jun 2023 10:10:06 +0000 Subject: [PATCH 33/41] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/solve_network.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index ea5446ad..41679a03 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -688,7 +688,9 @@ if __name__ == "__main__": config=snakemake.config, foresight_param=snakemake.params["foresight"], planning_horizons_param=snakemake.params["planning_horizons"], - co2_sequestration_potential_param=snakemake.params["co2_sequestration_potential"], + co2_sequestration_potential_param=snakemake.params[ + "co2_sequestration_potential" + ], ) n = solve_network( From 176bebdef0f12de8c639c83e70a9b949ae5c318e Mon Sep 17 00:00:00 2001 From: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Date: Fri, 9 Jun 2023 17:18:25 +0200 Subject: [PATCH 34/41] Update cluster_network to avoid adding deleted links in clustered network --- scripts/cluster_network.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 7572d3b3..d5d27ef7 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -424,7 +424,10 @@ def clustering_for_n_clusters( n.links.eval("underwater_fraction * length").div(nc.links.length).dropna() ) nc.links["capital_cost"] = nc.links["capital_cost"].add( - (nc.links.length - n.links.length).clip(lower=0).mul(extended_link_costs), + (nc.links.length - n.links.length) + .clip(lower=0) + .mul(extended_link_costs) + .dropna(), fill_value=0, ) From e44f8d66cdb2abb62a9ef59f1141b1895ce8d167 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 22:22:08 +0000 Subject: [PATCH 35/41] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/docformatter: v1.7.1 → v1.7.2](https://github.com/PyCQA/docformatter/compare/v1.7.1...v1.7.2) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 34d13746..826a4819 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: # Make docstrings PEP 257 compliant - repo: https://github.com/PyCQA/docformatter - rev: v1.7.1 + rev: v1.7.2 hooks: - id: docformatter args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"] From 12b93562453ccce90df248c2686040c984a1612b Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Wed, 14 Jun 2023 11:03:07 +0200 Subject: [PATCH 36/41] add info to release_notes --- doc/release_notes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 3af16477..8f26ac02 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,7 @@ Release Notes Upcoming Release ================ +* ``param:`` section in rule definition are added to track changed settings in ``config.yaml``. The goal is to automatically re-execute rules whose parameters have changed. See `Non-file parameters for rules `_ in the snakemake documentation. * **Important:** The configuration files are now located in the ``config`` directory. This counts for ``config.default.yaml``, ``config.yaml`` as well as the test configuration files which are now located in ``config/test``. Config files that are still in the root directory will be ignored. From 45cac01ea3c49529c4b68714e783bedb60565f00 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 14 Jun 2023 13:28:20 +0200 Subject: [PATCH 37/41] adjust param accessing in cluster, simplify and build_elec script --- rules/build_electricity.smk | 24 ++--- scripts/add_electricity.py | 40 ++++----- scripts/build_renewable_profiles.py | 2 +- scripts/cluster_network.py | 56 +++++------- scripts/simplify_network.py | 131 ++++++++++++---------------- 5 files changed, 109 insertions(+), 144 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 34276ba4..fd66cc04 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -316,16 +316,14 @@ rule add_electricity: rule simplify_network: params: - clustering=config["clustering"], - max_hours=config["electricity"]["max_hours"], + simplify_network=config["clustering"]["simplify_network"], + aggregation_strategies=config["clustering"].get("aggregation_strategies", {}), + focus_weights=config.get("focus_weights", None), + renewable_carriers=config["electricity"]["renewable_carriers"], costs=config["costs"], - renewable=config["renewable"], + max_hours=config["electricity"]["max_hours"], length_factor=config["lines"]["length_factor"], p_max_pu=config["links"].get("p_max_pu", 1.0), - exclude_carriers=config["clustering"]["simplify_network"].get( - "exclude_carriers", [] - ), - focus_weights=config.get("focus_weights", None), solver_name=config["solving"]["solver"]["name"], input: network=RESOURCES + "networks/elec.nc", @@ -353,14 +351,16 @@ rule simplify_network: rule cluster_network: params: - solver_name=config["solving"]["solver"]["name"], - max_hours=config["electricity"]["max_hours"], + cluster_network=config["clustering"]["cluster_network"], + aggregation_strategies=config["clustering"].get("aggregation_strategies", {}), + custom_busmap=config["enable"].get("custom_busmap", False), + focus_weights=config.get("focus_weights", None), + renewable_carriers=config["electricity"]["renewable_carriers"], conventional_carriers=config["electricity"].get("conventional_carriers", []), costs=config["costs"], + max_hours=config["electricity"]["max_hours"], length_factor=config["lines"]["length_factor"], - renewable=config["renewable"], - clustering=config["clustering"], - custom_busmap=config["enable"].get("custom_busmap", False), + solver_name=config["solving"]["solver"]["name"], input: network=RESOURCES + "networks/elec_s{simpl}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson", diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index dc5fb7cc..34147af3 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -726,23 +726,23 @@ if __name__ == "__main__": costs = load_costs( snakemake.input.tech_costs, - snakemake.params["costs"], - snakemake.params["electricity"]["max_hours"], + snakemake.params.costs, + snakemake.params.electricity["max_hours"], Nyears, ) ppl = load_powerplants(snakemake.input.powerplants) - if "renewable_carriers" in snakemake.params["electricity"]: - renewable_carriers = set(snakemake.params["electricity"]["renewable_carriers"]) + if "renewable_carriers" in snakemake.params.electricity: + renewable_carriers = set(snakemake.params.electricity["renewable_carriers"]) else: logger.warning( "Missing key `renewable_carriers` under config entry `electricity`. " "In future versions, this will raise an error. " "Falling back to carriers listed under `renewable`." ) - renewable_carriers = snakemake.params["renewable"] + renewable_carriers = snakemake.params.renewable - extendable_carriers = snakemake.params["electricity"]["extendable_carriers"] + extendable_carriers = snakemake.params.electricity["extendable_carriers"] if not (set(renewable_carriers) & set(extendable_carriers["Generator"])): logger.warning( "No renewables found in config entry `extendable_carriers`. " @@ -750,18 +750,18 @@ if __name__ == "__main__": "Falling back to all renewables." ) - conventional_carriers = snakemake.params["electricity"]["conventional_carriers"] + conventional_carriers = snakemake.params.electricity["conventional_carriers"] attach_load( n, snakemake.input.regions, snakemake.input.load, snakemake.input.nuts3_shapes, - snakemake.params["countries"], - snakemake.params["scaling_factor"], + snakemake.params.countries, + snakemake.params.scaling_factor, ) - update_transmission_costs(n, costs, snakemake.params["length_factor"]) + update_transmission_costs(n, costs, snakemake.params.length_factor) conventional_inputs = { k: v for k, v in snakemake.input.items() if k.startswith("conventional_") @@ -772,7 +772,7 @@ if __name__ == "__main__": ppl, conventional_carriers, extendable_carriers, - snakemake.params["conventional"], + snakemake.params.conventional, conventional_inputs, ) @@ -782,11 +782,11 @@ if __name__ == "__main__": snakemake.input, renewable_carriers, extendable_carriers, - snakemake.params["length_factor"], + snakemake.params.length_factor, ) if "hydro" in renewable_carriers: - para = snakemake.params["renewable"]["hydro"] + para = snakemake.params.renewable["hydro"] attach_hydro( n, costs, @@ -797,7 +797,7 @@ if __name__ == "__main__": **para, ) - if "estimate_renewable_capacities" not in snakemake.params["electricity"]: + if "estimate_renewable_capacities" not in snakemake.params.electricity: logger.warning( "Missing key `estimate_renewable_capacities` under config entry `electricity`. " "In future versions, this will raise an error. " @@ -805,18 +805,18 @@ if __name__ == "__main__": ) if ( "estimate_renewable_capacities_from_capacity_stats" - in snakemake.params["electricity"] + in snakemake.params.electricity ): estimate_renewable_caps = { "enable": True, - **snakemake.params["electricity"][ + **snakemake.params.electricity[ "estimate_renewable_capacities_from_capacity_stats" ], } else: estimate_renewable_caps = {"enable": False} else: - estimate_renewable_caps = snakemake.params["electricity"][ + estimate_renewable_caps = snakemake.params.electricity[ "estimate_renewable_capacities" ] if "enable" not in estimate_renewable_caps: @@ -832,18 +832,18 @@ if __name__ == "__main__": "Falling back to whether `renewable_capacities_from_opsd` is non-empty." ) from_opsd = bool( - snakemake.params["electricity"].get("renewable_capacities_from_opsd", False) + snakemake.params.electricity.get("renewable_capacities_from_opsd", False) ) estimate_renewable_caps["from_opsd"] = from_opsd if estimate_renewable_caps["enable"]: if estimate_renewable_caps["from_opsd"]: - tech_map = snakemake.params["electricity"]["estimate_renewable_capacities"][ + tech_map = snakemake.params.electricity["estimate_renewable_capacities"][ "technology_mapping" ] attach_OPSD_renewables(n, tech_map) estimate_renewable_capacities( - n, snakemake.params["electricity"], snakemake.params["countries"] + n, snakemake.params.electricity, snakemake.params.countries ) update_p_nom_max(n) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 91463c23..9e90d42b 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -309,7 +309,7 @@ if __name__ == "__main__": p_nom_max = capacities / max_cap_factor else: raise AssertionError( - 'params key `potential` should be one of "simple" ' + 'Config key `potential` should be one of "simple" ' f'(default) or "conservative", not "{p_nom_max_meth}"' ) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index db0fb2c2..52685af2 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -186,7 +186,7 @@ def get_feature_for_hac(n, buses_i=None, feature=None): if "offwind" in carriers: carriers.remove("offwind") carriers = np.append( - carriers, network.generators.carrier.filter(like="offwind").unique() + carriers, n.generators.carrier.filter(like="offwind").unique() ) if feature.split("-")[1] == "cap": @@ -463,26 +463,18 @@ if __name__ == "__main__": snakemake = mock_snakemake("cluster_network", simpl="", clusters="5") configure_logging(snakemake) + params = snakemake.params + solver_name = snakemake.config["solving"]["solver"]["name"] + n = pypsa.Network(snakemake.input.network) - focus_weights = snakemake.config.get("focus_weights", None) - - renewable_carriers = pd.Index( - [ - tech - for tech in n.generators.carrier.unique() - if tech in snakemake.params["renewable"] - ] - ) - - exclude_carriers = snakemake.params["clustering"]["cluster_network"].get( - "exclude_carriers", [] - ) + exclude_carriers = params.cluster_network["exclude_carriers"] aggregate_carriers = set(n.generators.carrier) - set(exclude_carriers) if snakemake.wildcards.clusters.endswith("m"): n_clusters = int(snakemake.wildcards.clusters[:-1]) - conventional = set(snakemake.params["conventional_carriers"]) - aggregate_carriers = conventional.intersection(aggregate_carriers) + aggregate_carriers = set(params.conventional_carriers).intersection( + aggregate_carriers + ) elif snakemake.wildcards.clusters == "all": n_clusters = len(n.buses) else: @@ -496,13 +488,12 @@ if __name__ == "__main__": n, busmap, linemap, linemap, pd.Series(dtype="O") ) else: - line_length_factor = snakemake.params["length_factor"] Nyears = n.snapshot_weightings.objective.sum() / 8760 hvac_overhead_cost = load_costs( snakemake.input.tech_costs, - snakemake.params["costs"], - snakemake.params["max_hours"], + params.costs, + params.max_hours, Nyears, ).at["HVAC overhead", "capital_cost"] @@ -513,16 +504,16 @@ if __name__ == "__main__": ).all() or x.isnull().all(), "The `potential` configuration option must agree for all renewable carriers, for now!" return v - aggregation_strategies = snakemake.params["clustering"].get( - "aggregation_strategies", {} - ) # translate str entries of aggregation_strategies to pd.Series functions: aggregation_strategies = { - p: {k: getattr(pd.Series, v) for k, v in aggregation_strategies[p].items()} - for p in aggregation_strategies.keys() + p: { + k: getattr(pd.Series, v) + for k, v in params.aggregation_strategies[p].items() + } + for p in params.aggregation_strategies.keys() } - custom_busmap = snakemake.params["custom_busmap"] + custom_busmap = params.custom_busmap if custom_busmap: custom_busmap = pd.read_csv( snakemake.input.custom_busmap, index_col=0, squeeze=True @@ -530,21 +521,18 @@ if __name__ == "__main__": custom_busmap.index = custom_busmap.index.astype(str) logger.info(f"Imported custom busmap from {snakemake.input.custom_busmap}") - cluster_config = snakemake.config.get("clustering", {}).get( - "cluster_network", {} - ) clustering = clustering_for_n_clusters( n, n_clusters, custom_busmap, aggregate_carriers, - line_length_factor, - aggregation_strategies, - snakemake.params["solver_name"], - cluster_config.get("algorithm", "hac"), - cluster_config.get("feature", "solar+onwind-time"), + params.length_factor, + params.aggregation_strategies, + solver_name, + params.cluster_network["algorithm"], + params.cluster_network["feature"], hvac_overhead_cost, - focus_weights, + params.focus_weights, ) update_p_nom_max(clustering.network) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 83e932a3..79161760 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -149,17 +149,17 @@ def simplify_network_to_380(n): return n, trafo_map -def _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param): +def _prepare_connection_costs_per_link(n, costs, renewable_carriers, length_factor): if n.links.empty: return {} connection_costs_per_link = {} - for tech in renewable_param: + for tech in renewable_carriers: if tech.startswith("offwind"): connection_costs_per_link[tech] = ( n.links.length - * length_factor_param + * length_factor * ( n.links.underwater_fraction * costs.at[tech + "-connection-submarine", "capital_cost"] @@ -175,14 +175,14 @@ def _compute_connection_costs_to_bus( n, busmap, costs, - renewable_param, - length_factor_param, + renewable_carriers, + length_factor, connection_costs_per_link=None, buses=None, ): if connection_costs_per_link is None: connection_costs_per_link = _prepare_connection_costs_per_link( - n, costs, renewable_param, length_factor_param + n, costs, renewable_carriers, length_factor ) if buses is None: @@ -276,10 +276,10 @@ def _aggregate_and_move_components( def simplify_links( n, costs, - renewable_param, - length_factor_param, - p_max_pu_param, - exclude_carriers_param, + renewables, + length_factor, + p_max_pu, + exclude_carriers, output, aggregation_strategies=dict(), ): @@ -333,7 +333,7 @@ def simplify_links( busmap = n.buses.index.to_series() connection_costs_per_link = _prepare_connection_costs_per_link( - n, costs, renewable_param, length_factor_param + n, costs, renewables, length_factor ) connection_costs_to_bus = pd.DataFrame( 0.0, index=n.buses.index, columns=list(connection_costs_per_link) @@ -355,8 +355,8 @@ def simplify_links( n, busmap, costs, - renewable_param, - length_factor_param, + renewables, + length_factor, connection_costs_per_link, buses, ) @@ -378,8 +378,8 @@ def simplify_links( / lengths.sum() * n.links.loc[all_links, "underwater_fraction"] ), - p_max_pu=p_max_pu_param, - p_min_pu=-p_max_pu_param, + p_max_pu=p_max_pu, + p_min_pu=-p_max_pu, underground=False, under_construction=False, ) @@ -407,7 +407,7 @@ def simplify_links( connection_costs_to_bus, output, aggregation_strategies=aggregation_strategies, - exclude_carriers=exclude_carriers_param, + exclude_carriers=exclude_carriers, ) return n, busmap @@ -415,34 +415,29 @@ def simplify_links( def remove_stubs( n, costs, - renewable_param, - length_factor_param, - clustering_param, - exclude_carriers_param, + renewable_carriers, + length_factor, + simplify_network, output, aggregation_strategies=dict(), ): logger.info("Removing stubs") - across_borders = clustering_param["simplify_network"].get( - "remove_stubs_across_borders", True - ) + across_borders = simplify_network["remove_stubs_across_borders"] matching_attrs = [] if across_borders else ["country"] busmap = busmap_by_stubs(n, matching_attrs) connection_costs_to_bus = _compute_connection_costs_to_bus( - n, busmap, costs, renewable_param, length_factor_param + n, busmap, costs, renewable_carriers, length_factor ) - exclude_carriers = clustering_param["simplify_network"].get("exclude_carriers", []) - _aggregate_and_move_components( n, busmap, connection_costs_to_bus, output, aggregation_strategies=aggregation_strategies, - exclude_carriers=exclude_carriers, + exclude_carriers=simplify_network["exclude_carriers"], ) return n, busmap @@ -504,32 +499,23 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None): def cluster( n, n_clusters, - focus_weights_param, - renewable_param, - solver_name_param, + focus_weights, + solver_name, algorithm="hac", feature=None, aggregation_strategies=dict(), ): logger.info(f"Clustering to {n_clusters} buses") - renewable_carriers = pd.Index( - [ - tech - for tech in n.generators.carrier.unique() - if tech.split("-", 2)[0] in renewable_param - ] - ) - clustering = clustering_for_n_clusters( n, n_clusters, custom_busmap=False, aggregation_strategies=aggregation_strategies, - solver_name=solver_name_param, + solver_name=solver_name, algorithm=algorithm, feature=feature, - focus_weights=focus_weights_param, + focus_weights=focus_weights, ) return clustering.network, clustering.busmap @@ -542,77 +528,69 @@ if __name__ == "__main__": snakemake = mock_snakemake("simplify_network", simpl="") configure_logging(snakemake) - n = pypsa.Network(snakemake.input.network) + params = snakemake.params + solver_name = snakemake.config["solving"]["solver"]["name"] + + n = pypsa.Network(snakemake.input.network) + Nyears = n.snapshot_weightings.objective.sum() / 8760 - aggregation_strategies = snakemake.params["clustering"].get( - "aggregation_strategies", {} - ) # translate str entries of aggregation_strategies to pd.Series functions: aggregation_strategies = { - p: {k: getattr(pd.Series, v) for k, v in aggregation_strategies[p].items()} - for p in aggregation_strategies.keys() + p: { + k: getattr(pd.Series, v) + for k, v in params.aggregation_strategies[p].items() + } + for p in params.aggregation_strategies.keys() } n, trafo_map = simplify_network_to_380(n) - Nyears = n.snapshot_weightings.objective.sum() / 8760 - technology_costs = load_costs( snakemake.input.tech_costs, - snakemake.params["costs"], - snakemake.params["max_hours"], + params.costs, + params.max_hours, Nyears, ) n, simplify_links_map = simplify_links( n, technology_costs, - snakemake.params["renewable"], - snakemake.params["length_factor"], - snakemake.params["p_max_pu"], - snakemake.params["exclude_carriers"], + params.renewable_carriers, + params.length_factor, + params.p_max_pu, + params.simplify_network["exclude_carriers"], snakemake.output, aggregation_strategies, ) busmaps = [trafo_map, simplify_links_map] - cluster_param = snakemake.params["clustering"]["simplify_network"] - if cluster_param.get("remove_stubs", True): + if params.simplify_network["remove_stubs"]: n, stub_map = remove_stubs( n, technology_costs, - snakemake.params["renewable"], - snakemake.params["length_factor"], - snakemake.params["clustering"], - snakemake.params["exclude_carriers"], + params.renewable_carriers, + params.length_factor, + params.simplify_network, snakemake.output, aggregation_strategies=aggregation_strategies, ) busmaps.append(stub_map) - if cluster_param.get("to_substations", False): + if params.simplify_network["to_substations"]: n, substation_map = aggregate_to_substations(n, aggregation_strategies) busmaps.append(substation_map) # treatment of outliers (nodes without a profile for considered carrier): # all nodes that have no profile of the given carrier are being aggregated to closest neighbor - if ( - snakemake.config.get("clustering", {}) - .get("cluster_network", {}) - .get("algorithm", "hac") - == "hac" - or cluster_param.get("algorithm", "hac") == "hac" - ): - carriers = ( - cluster_param.get("feature", "solar+onwind-time").split("-")[0].split("+") - ) + if params.simplify_network["algorithm"] == "hac": + carriers = params.simplify_network["feature"].split("-")[0].split("+") for carrier in carriers: buses_i = list( set(n.buses.index) - set(n.generators.query("carrier == @carrier").bus) ) logger.info( - f"clustering preparaton (hac): aggregating {len(buses_i)} buses of type {carrier}." + f"clustering preparation (hac): aggregating {len(buses_i)} buses of type {carrier}." ) n, busmap_hac = aggregate_to_substations(n, aggregation_strategies, buses_i) busmaps.append(busmap_hac) @@ -621,11 +599,10 @@ if __name__ == "__main__": n, cluster_map = cluster( n, int(snakemake.wildcards.simpl), - snakemake.params["focus_weights"], - snakemake.params["renewable"], - snakemake.params["solver_name"], - cluster_param.get("algorithm", "hac"), - cluster_param.get("feature", None), + params.focus_weights, + solver_name, + params.simplify_network["algorithm"], + params.simplify_network["feature"], aggregation_strategies, ) busmaps.append(cluster_map) From 1d10073514a66f09e6821364f03a57f99cff2352 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 15 Jun 2023 18:52:25 +0200 Subject: [PATCH 38/41] review params implementation; reproduce objective values in CI --- config/test/config.myopic.yaml | 8 ++ config/test/config.overnight.yaml | 8 ++ rules/build_electricity.smk | 2 +- rules/build_sector.smk | 2 +- scripts/add_brownfield.py | 6 +- scripts/add_electricity.py | 93 +++++-------------- scripts/add_existing_baseyear.py | 20 ++-- scripts/add_extra_components.py | 26 +++--- scripts/build_ammonia_production.py | 2 +- scripts/build_biomass_potentials.py | 2 +- scripts/build_bus_regions.py | 2 +- scripts/build_cop_profiles.py | 2 +- scripts/build_cutout.py | 4 +- scripts/build_electricity_demand.py | 12 +-- scripts/build_energy_totals.py | 8 +- scripts/build_heat_demand.py | 2 +- scripts/build_hydro_profile.py | 4 +- scripts/build_industrial_distribution_key.py | 4 +- ...ustrial_energy_demand_per_country_today.py | 4 +- ...build_industrial_production_per_country.py | 6 +- ...ustrial_production_per_country_tomorrow.py | 2 +- scripts/build_industry_sector_ratios.py | 4 +- scripts/build_powerplants.py | 6 +- scripts/build_renewable_profiles.py | 2 +- scripts/build_retro_cost.py | 4 +- scripts/build_sequestration_potentials.py | 2 +- scripts/build_shapes.py | 6 +- scripts/build_solar_thermal_profiles.py | 4 +- scripts/build_temperature_profiles.py | 2 +- scripts/build_transport_demand.py | 4 +- scripts/make_summary.py | 20 ++-- scripts/plot_network.py | 16 ++-- scripts/plot_summary.py | 24 ++--- scripts/prepare_network.py | 20 ++-- scripts/prepare_sector_network.py | 40 ++++---- scripts/retrieve_databundle.py | 2 +- scripts/solve_network.py | 42 ++++----- scripts/solve_operations_network.py | 2 +- 38 files changed, 188 insertions(+), 231 deletions(-) diff --git a/config/test/config.myopic.yaml b/config/test/config.myopic.yaml index efa03136..0bb85ec6 100644 --- a/config/test/config.myopic.yaml +++ b/config/test/config.myopic.yaml @@ -31,6 +31,14 @@ snapshots: end: "2013-03-08" electricity: + co2limit: 100.e+6 + + extendable_carriers: + Generator: [OCGT] + StorageUnit: [battery] + Store: [H2] + Link: [H2 pipeline] + renewable_carriers: [solar, onwind, offwind-ac, offwind-dc] atlite: diff --git a/config/test/config.overnight.yaml b/config/test/config.overnight.yaml index fb468ded..a2a0f5a4 100644 --- a/config/test/config.overnight.yaml +++ b/config/test/config.overnight.yaml @@ -28,6 +28,14 @@ snapshots: end: "2013-03-08" electricity: + co2limit: 100.e+6 + + extendable_carriers: + Generator: [OCGT] + StorageUnit: [battery] + Store: [H2] + Link: [H2 pipeline] + renewable_carriers: [solar, onwind, offwind-ac, offwind-dc] atlite: diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index fd66cc04..9d4315df 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -394,7 +394,7 @@ rule cluster_network: rule add_extra_components: params: costs=config["costs"], - ext_carriers=config["electricity"]["extendable_carriers"], + extendable_carriers=config["electricity"]["extendable_carriers"], max_hours=config["electricity"]["max_hours"], input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 3714cbad..e15394ff 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -406,7 +406,7 @@ rule build_ammonia_production: rule build_industry_sector_ratios: params: industry=config["industry"], - sector_amonia=config["sector"].get("ammonia", False), + ammonia=config["sector"].get("ammonia", False), input: ammonia_production=RESOURCES + "ammonia_production.csv", idees="data/jrc-idees-2015", diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 48dffbb9..b33d2f19 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -49,7 +49,7 @@ def add_brownfield(n, n_p, year): ) ] - threshold = snakemake.params["threshold_capacity"] + threshold = snakemake.params.threshold_capacity if not chp_heat.empty: threshold_chp_heat = ( @@ -87,7 +87,7 @@ def add_brownfield(n, n_p, year): # deal with gas network pipe_carrier = ["gas pipeline"] - if snakemake.params["H2_retrofit"]: + if snakemake.params.H2_retrofit: # drop capacities of previous year to avoid duplicating to_drop = n.links.carrier.isin(pipe_carrier) & (n.links.build_year != year) n.mremove("Link", n.links.loc[to_drop].index) @@ -98,7 +98,7 @@ def add_brownfield(n, n_p, year): & (n.links.build_year != year) ].index gas_pipes_i = n.links[n.links.carrier.isin(pipe_carrier)].index - CH4_per_H2 = 1 / snakemake.params["H2_retrofit_capacity_per_CH4"] + CH4_per_H2 = 1 / snakemake.params.H2_retrofit_capacity_per_CH4 fr = "H2 pipeline retrofitted" to = "gas pipeline" # today's pipe capacity diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 34147af3..d98dc767 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -652,13 +652,7 @@ def attach_OPSD_renewables(n, tech_map): n.generators.p_nom_min.update(gens.bus.map(caps).dropna()) -def estimate_renewable_capacities(n, electricity_params, countries): - year = electricity_params["estimate_renewable_capacities"]["year"] - tech_map = electricity_params["estimate_renewable_capacities"]["technology_mapping"] - expansion_limit = electricity_params["estimate_renewable_capacities"][ - "expansion_limit" - ] - +def estimate_renewable_capacities(n, year, tech_map, expansion_limit, countries): if not len(countries) or not len(tech_map): return @@ -721,48 +715,42 @@ if __name__ == "__main__": snakemake = mock_snakemake("add_electricity") configure_logging(snakemake) + params = snakemake.params + n = pypsa.Network(snakemake.input.base_network) Nyears = n.snapshot_weightings.objective.sum() / 8760.0 costs = load_costs( snakemake.input.tech_costs, - snakemake.params.costs, - snakemake.params.electricity["max_hours"], + params.costs, + params.electricity["max_hours"], Nyears, ) ppl = load_powerplants(snakemake.input.powerplants) - if "renewable_carriers" in snakemake.params.electricity: - renewable_carriers = set(snakemake.params.electricity["renewable_carriers"]) + if "renewable_carriers" in params.electricity: + renewable_carriers = set(params.electricity["renewable_carriers"]) else: logger.warning( "Missing key `renewable_carriers` under config entry `electricity`. " "In future versions, this will raise an error. " "Falling back to carriers listed under `renewable`." ) - renewable_carriers = snakemake.params.renewable - - extendable_carriers = snakemake.params.electricity["extendable_carriers"] - if not (set(renewable_carriers) & set(extendable_carriers["Generator"])): - logger.warning( - "No renewables found in config entry `extendable_carriers`. " - "In future versions, these have to be explicitly listed. " - "Falling back to all renewables." - ) - - conventional_carriers = snakemake.params.electricity["conventional_carriers"] + renewable_carriers = params.renewable attach_load( n, snakemake.input.regions, snakemake.input.load, snakemake.input.nuts3_shapes, - snakemake.params.countries, - snakemake.params.scaling_factor, + params.countries, + params.scaling_factor, ) - update_transmission_costs(n, costs, snakemake.params.length_factor) + update_transmission_costs(n, costs, params.length_factor) + extendable_carriers = params.electricity["extendable_carriers"] + conventional_carriers = params.electricity["conventional_carriers"] conventional_inputs = { k: v for k, v in snakemake.input.items() if k.startswith("conventional_") } @@ -772,7 +760,7 @@ if __name__ == "__main__": ppl, conventional_carriers, extendable_carriers, - snakemake.params.conventional, + params.conventional, conventional_inputs, ) @@ -782,11 +770,11 @@ if __name__ == "__main__": snakemake.input, renewable_carriers, extendable_carriers, - snakemake.params.length_factor, + params.length_factor, ) if "hydro" in renewable_carriers: - para = snakemake.params.renewable["hydro"] + para = params.renewable["hydro"] attach_hydro( n, costs, @@ -797,53 +785,16 @@ if __name__ == "__main__": **para, ) - if "estimate_renewable_capacities" not in snakemake.params.electricity: - logger.warning( - "Missing key `estimate_renewable_capacities` under config entry `electricity`. " - "In future versions, this will raise an error. " - "Falling back to whether ``estimate_renewable_capacities_from_capacity_stats`` is in the config." - ) - if ( - "estimate_renewable_capacities_from_capacity_stats" - in snakemake.params.electricity - ): - estimate_renewable_caps = { - "enable": True, - **snakemake.params.electricity[ - "estimate_renewable_capacities_from_capacity_stats" - ], - } - else: - estimate_renewable_caps = {"enable": False} - else: - estimate_renewable_caps = snakemake.params.electricity[ - "estimate_renewable_capacities" - ] - if "enable" not in estimate_renewable_caps: - logger.warning( - "Missing key `enable` under config entry `estimate_renewable_capacities`. " - "In future versions, this will raise an error. Falling back to False." - ) - estimate_renewable_caps = {"enable": False} - if "from_opsd" not in estimate_renewable_caps: - logger.warning( - "Missing key `from_opsd` under config entry `estimate_renewable_capacities`. " - "In future versions, this will raise an error. " - "Falling back to whether `renewable_capacities_from_opsd` is non-empty." - ) - from_opsd = bool( - snakemake.params.electricity.get("renewable_capacities_from_opsd", False) - ) - estimate_renewable_caps["from_opsd"] = from_opsd - + estimate_renewable_caps = params.electricity["estimate_renewable_capacities"] if estimate_renewable_caps["enable"]: + tech_map = estimate_renewable_caps["technology_mapping"] + expansion_limit = estimate_renewable_caps["expansion_limit"] + year = estimate_renewable_caps["year"] + if estimate_renewable_caps["from_opsd"]: - tech_map = snakemake.params.electricity["estimate_renewable_capacities"][ - "technology_mapping" - ] attach_OPSD_renewables(n, tech_map) estimate_renewable_capacities( - n, snakemake.params.electricity, snakemake.params.countries + n, year, tech_map, expansion_limit, params.countries ) update_p_nom_max(n) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index bc1d3a05..52cb585e 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -157,7 +157,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas # Fill missing DateOut dateout = ( df_agg.loc[biomass_i, "DateIn"] - + snakemake.params["costs"]["fill_values"]["lifetime"] + + snakemake.params.costs["fill_values"]["lifetime"] ) df_agg.loc[biomass_i, "DateOut"] = df_agg.loc[biomass_i, "DateOut"].fillna(dateout) @@ -218,7 +218,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas capacity = df.loc[grouping_year, generator] capacity = capacity[~capacity.isna()] capacity = capacity[ - capacity > snakemake.params["existing_capacities"]["threshold_capacity"] + capacity > snakemake.params.existing_capacities["threshold_capacity"] ] suffix = "-ac" if generator == "offwind" else "" name_suffix = f" {generator}{suffix}-{grouping_year}" @@ -582,7 +582,7 @@ def add_heating_capacities_installed_before_baseyear( ) # delete links with capacities below threshold - threshold = snakemake.params["existing_capacities"]["threshold_capacity"] + threshold = snakemake.params.existing_capacities["threshold_capacity"] n.mremove( "Link", [ @@ -612,10 +612,10 @@ if __name__ == "__main__": update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) - options = snakemake.params["sector"] + options = snakemake.params.sector opts = snakemake.wildcards.sector_opts.split("-") - baseyear = snakemake.params["baseyear"] + baseyear = snakemake.params.baseyear overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) @@ -626,14 +626,12 @@ if __name__ == "__main__": Nyears = n.snapshot_weightings.generators.sum() / 8760.0 costs = prepare_costs( snakemake.input.costs, - snakemake.params["costs"], + snakemake.params.costs, Nyears, ) - grouping_years_power = snakemake.params["existing_capacities"][ - "grouping_years_power" - ] - grouping_years_heat = snakemake.params["existing_capacities"]["grouping_years_heat"] + grouping_years_power = snakemake.params.existing_capacities["grouping_years_power"] + grouping_years_heat = snakemake.params.existing_capacities["grouping_years_heat"] add_power_capacities_installed_before_baseyear( n, grouping_years_power, costs, baseyear ) @@ -650,7 +648,7 @@ if __name__ == "__main__": .to_pandas() .reindex(index=n.snapshots) ) - default_lifetime = snakemake.params["costs"]["fill_values"]["lifetime"] + default_lifetime = snakemake.params.costs["fill_values"]["lifetime"] add_heating_capacities_installed_before_baseyear( n, baseyear, diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index ce4e533e..f034a7f2 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -67,8 +67,8 @@ idx = pd.IndexSlice logger = logging.getLogger(__name__) -def attach_storageunits(n, costs, ext_carriers, max_hours): - carriers = ext_carriers["StorageUnit"] +def attach_storageunits(n, costs, extendable_carriers, max_hours): + carriers = extendable_carriers["StorageUnit"] _add_missing_carriers_from_costs(n, costs, carriers) @@ -98,8 +98,8 @@ def attach_storageunits(n, costs, ext_carriers, max_hours): ) -def attach_stores(n, costs, ext_carriers): - carriers = ext_carriers["Store"] +def attach_stores(n, costs, extendable_carriers): + carriers = extendable_carriers["Store"] _add_missing_carriers_from_costs(n, costs, carriers) @@ -186,10 +186,10 @@ def attach_stores(n, costs, ext_carriers): ) -def attach_hydrogen_pipelines(n, costs, ext_carriers): - as_stores = ext_carriers.get("Store", []) +def attach_hydrogen_pipelines(n, costs, extendable_carriers): + as_stores = extendable_carriers.get("Store", []) - if "H2 pipeline" not in ext_carriers.get("Link", []): + if "H2 pipeline" not in extendable_carriers.get("Link", []): return assert "H2" in as_stores, ( @@ -233,17 +233,17 @@ if __name__ == "__main__": configure_logging(snakemake) n = pypsa.Network(snakemake.input.network) - ext_carriers = snakemake.params["ext_carriers"] - max_hours = snakemake.params["max_hours"] + extendable_carriers = snakemake.params.extendable_carriers + max_hours = snakemake.params.max_hours Nyears = n.snapshot_weightings.objective.sum() / 8760.0 costs = load_costs( - snakemake.input.tech_costs, snakemake.params["costs"], max_hours, Nyears + snakemake.input.tech_costs, snakemake.params.costs, max_hours, Nyears ) - attach_storageunits(n, costs, ext_carriers, max_hours) - attach_stores(n, costs, ext_carriers) - attach_hydrogen_pipelines(n, costs, ext_carriers) + attach_storageunits(n, costs, extendable_carriers, max_hours) + attach_stores(n, costs, extendable_carriers) + attach_hydrogen_pipelines(n, costs, extendable_carriers) add_nice_carrier_names(n, snakemake.config) diff --git a/scripts/build_ammonia_production.py b/scripts/build_ammonia_production.py index 6f03324f..1bcdf9ae 100644 --- a/scripts/build_ammonia_production.py +++ b/scripts/build_ammonia_production.py @@ -30,7 +30,7 @@ if __name__ == "__main__": ammonia.index = cc.convert(ammonia.index, to="iso2") years = [str(i) for i in range(2013, 2018)] - countries = ammonia.index.intersection(snakemake.params["countries"]) + countries = ammonia.index.intersection(snakemake.params.countries) ammonia = ammonia.loc[countries, years].astype(float) # convert from ktonN to ktonNH3 diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index 0b423ad5..d200a78e 100644 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -210,7 +210,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_biomass_potentials", simpl="", clusters="5") - params = snakemake.params["biomass"] + params = snakemake.params.biomass year = params["year"] scenario = params["scenario"] diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index e9378792..a6500bb0 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -116,7 +116,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_bus_regions") configure_logging(snakemake) - countries = snakemake.params["countries"] + countries = snakemake.params.countries n = pypsa.Network(snakemake.input.base_network) diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py index 983eda2d..4b1d952e 100644 --- a/scripts/build_cop_profiles.py +++ b/scripts/build_cop_profiles.py @@ -39,7 +39,7 @@ if __name__ == "__main__": for source in ["air", "soil"]: source_T = xr.open_dataarray(snakemake.input[f"temp_{source}_{area}"]) - delta_T = snakemake.params["heat_pump_sink_T"] - source_T + delta_T = snakemake.params.heat_pump_sink_T - source_T cop = coefficient_of_performance(delta_T, source) diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 798588d9..9a7f9e00 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -106,9 +106,9 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_cutout", cutout="europe-2013-era5") configure_logging(snakemake) - cutout_params = snakemake.params["cutouts"][snakemake.wildcards.cutout] + cutout_params = snakemake.params.cutouts[snakemake.wildcards.cutout] - snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"]) + snapshots = pd.date_range(freq="h", **snakemake.params.snapshots) time = [snapshots[0], snapshots[-1]] cutout_params["time"] = slice(*cutout_params.get("time", time)) diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py index 4ef56d1d..ba1fb881 100755 --- a/scripts/build_electricity_demand.py +++ b/scripts/build_electricity_demand.py @@ -279,16 +279,16 @@ if __name__ == "__main__": configure_logging(snakemake) - powerstatistics = snakemake.params["load"]["power_statistics"] - interpolate_limit = snakemake.params["load"]["interpolate_limit"] - countries = snakemake.params["countries"] - snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"]) + powerstatistics = snakemake.params.load["power_statistics"] + interpolate_limit = snakemake.params.load["interpolate_limit"] + countries = snakemake.params.countries + snapshots = pd.date_range(freq="h", **snakemake.params.snapshots) years = slice(snapshots[0], snapshots[-1]) - time_shift = snakemake.params["load"]["time_shift_for_large_gaps"] + time_shift = snakemake.params.load["time_shift_for_large_gaps"] load = load_timeseries(snakemake.input[0], years, countries, powerstatistics) - if snakemake.params["load"]["manual_adjustments"]: + if snakemake.params.load["manual_adjustments"]: load = manual_adjustment(load, snakemake.input[0], powerstatistics) logger.info(f"Linearly interpolate gaps of size {interpolate_limit} and less.") diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 3e3cb485..891c4e2a 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -737,16 +737,16 @@ if __name__ == "__main__": logging.basicConfig(level=snakemake.config["logging"]["level"]) - params = snakemake.params["energy"] + params = snakemake.params.energy nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index("index") population = nuts3["pop"].groupby(nuts3.country).sum() - countries = snakemake.params["countries"] + countries = snakemake.params.countries idees_countries = pd.Index(countries).intersection(eu28) data_year = params["energy_totals_year"] - report_year = snakemake.params["energy"]["eurostat_report_year"] + report_year = snakemake.params.energy["eurostat_report_year"] input_eurostat = snakemake.input.eurostat eurostat = build_eurostat(input_eurostat, countries, report_year, data_year) swiss = build_swiss(data_year) @@ -756,7 +756,7 @@ if __name__ == "__main__": energy.to_csv(snakemake.output.energy_name) base_year_emissions = params["base_emissions_year"] - emissions_scope = snakemake.params["energy"]["emissions"] + emissions_scope = snakemake.params.energy["emissions"] eea_co2 = build_eea_co2(snakemake.input.co2, base_year_emissions, emissions_scope) eurostat_co2 = build_eurostat_co2( input_eurostat, countries, report_year, base_year_emissions diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py index 655df28f..73494260 100644 --- a/scripts/build_heat_demand.py +++ b/scripts/build_heat_demand.py @@ -27,7 +27,7 @@ if __name__ == "__main__": cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - time = pd.date_range(freq="h", **snakemake.params["snapshots"]) + time = pd.date_range(freq="h", **snakemake.params.snapshots) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) clustered_regions = ( diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 26bf31c6..bed666f2 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -130,10 +130,10 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_hydro_profile") configure_logging(snakemake) - params_hydro = snakemake.params["hydro"] + params_hydro = snakemake.params.hydro cutout = atlite.Cutout(snakemake.input.cutout) - countries = snakemake.params["countries"] + countries = snakemake.params.countries country_shapes = ( gpd.read_file(snakemake.input.country_shapes) .set_index("name")["geometry"] diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index a51ff8b2..e93e43c2 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -73,7 +73,7 @@ def prepare_hotmaps_database(regions): df[["srid", "coordinates"]] = df.geom.str.split(";", expand=True) - if snakemake.params["hotmaps_locate_missing"]: + if snakemake.params.hotmaps_locate_missing: df = locate_missing_industrial_sites(df) # remove those sites without valid locations @@ -143,7 +143,7 @@ if __name__ == "__main__": logging.basicConfig(level=snakemake.config["logging"]["level"]) - countries = snakemake.params["countries"] + countries = snakemake.params.countries regions = gpd.read_file(snakemake.input.regions_onshore).set_index("name") diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 9f8c47d0..9ca0d003 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -178,9 +178,9 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_industrial_energy_demand_per_country_today") - params = snakemake.params["industry"] + params = snakemake.params.industry year = params.get("reference_year", 2015) - countries = pd.Index(snakemake.params["countries"]) + countries = pd.Index(snakemake.params.countries) demand = industrial_energy_demand(countries.intersection(eu28), year) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 889c9ecd..74cb1949 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -279,11 +279,11 @@ if __name__ == "__main__": logging.basicConfig(level=snakemake.config["logging"]["level"]) - countries = snakemake.params["countries"] + countries = snakemake.params.countries - year = snakemake.params["industry"]["reference_year"] + year = snakemake.params.industry["reference_year"] - params = snakemake.params["industry"] + params = snakemake.params.industry jrc_dir = snakemake.input.jrc eurostat_dir = snakemake.input.eurostat diff --git a/scripts/build_industrial_production_per_country_tomorrow.py b/scripts/build_industrial_production_per_country_tomorrow.py index 609170aa..ffed5195 100644 --- a/scripts/build_industrial_production_per_country_tomorrow.py +++ b/scripts/build_industrial_production_per_country_tomorrow.py @@ -15,7 +15,7 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_industrial_production_per_country_tomorrow") - params = snakemake.params["industry"] + params = snakemake.params.industry investment_year = int(snakemake.wildcards.planning_horizons) diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index fa9e5c18..45705002 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -439,7 +439,7 @@ def chemicals_industry(): sector = "Ammonia" df[sector] = 0.0 - if snakemake.params["sector_amonia"]: + if snakemake.params.ammonia: df.loc["ammonia", sector] = params["MWh_NH3_per_tNH3"] else: df.loc["hydrogen", sector] = params["MWh_H2_per_tNH3_electrolysis"] @@ -1468,7 +1468,7 @@ if __name__ == "__main__": # TODO make params option year = 2015 - params = snakemake.params["industry"] + params = snakemake.params.industry df = pd.concat( [ diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index 6edd4ac4..7f396e1e 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -115,7 +115,7 @@ if __name__ == "__main__": configure_logging(snakemake) n = pypsa.Network(snakemake.input.base_network) - countries = snakemake.params["countries"] + countries = snakemake.params.countries ppl = ( pm.powerplants(from_url=True) @@ -134,12 +134,12 @@ if __name__ == "__main__": ppl = ppl.query('not (Country in @available_countries and Fueltype == "Bioenergy")') ppl = pd.concat([ppl, opsd]) - ppl_query = snakemake.params["powerplants_filter"] + ppl_query = snakemake.params.powerplants_filter if isinstance(ppl_query, str): ppl.query(ppl_query, inplace=True) # add carriers from own powerplant files: - custom_ppl_query = snakemake.params["custom_powerplants"] + custom_ppl_query = snakemake.params.custom_powerplants ppl = add_custom_powerplants( ppl, snakemake.input.custom_powerplants, custom_ppl_query ) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 9e90d42b..9f1f1a51 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -204,7 +204,7 @@ if __name__ == "__main__": nprocesses = int(snakemake.threads) noprogress = snakemake.config["run"].get("disable_progressbar", True) - params = snakemake.params["renewable"][snakemake.wildcards.technology] + params = snakemake.params.renewable[snakemake.wildcards.technology] resource = params["resource"] # pv panel params / wind turbine params correction_factor = params.get("correction_factor", 1.0) capacity_per_sqkm = params["capacity_per_sqkm"] diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index ac7eb4ae..c830415e 100644 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -305,7 +305,7 @@ def prepare_building_stock_data(): u_values.set_index(["country_code", "subsector", "bage", "type"], inplace=True) # only take in config.yaml specified countries into account - countries = snakemake.params["countries"] + countries = snakemake.params.countries area_tot = area_tot.loc[countries] return u_values, country_iso_dic, countries, area_tot, area @@ -1040,7 +1040,7 @@ if __name__ == "__main__": # ******** config ********************************************************* - retro_opts = snakemake.params["retrofitting"] + retro_opts = snakemake.params.retrofitting interest_rate = retro_opts["interest_rate"] annualise_cost = retro_opts["annualise_cost"] # annualise the investment costs tax_weighting = retro_opts[ diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py index 9d26b0b9..0e59e55b 100644 --- a/scripts/build_sequestration_potentials.py +++ b/scripts/build_sequestration_potentials.py @@ -41,7 +41,7 @@ if __name__ == "__main__": "build_sequestration_potentials", simpl="", clusters="181" ) - cf = snakemake.params["co2seq_potential"] + cf = snakemake.params.co2seq_potential gdf = gpd.read_file(snakemake.input.sequestration_potential[0]) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 0c8b0a94..eb837409 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -255,13 +255,11 @@ if __name__ == "__main__": snakemake = mock_snakemake("build_shapes") configure_logging(snakemake) - country_shapes = countries( - snakemake.input.naturalearth, snakemake.params["countries"] - ) + country_shapes = countries(snakemake.input.naturalearth, snakemake.params.countries) country_shapes.reset_index().to_file(snakemake.output.country_shapes) offshore_shapes = eez( - country_shapes, snakemake.input.eez, snakemake.params["countries"] + country_shapes, snakemake.input.eez, snakemake.params.countries ) offshore_shapes.reset_index().to_file(snakemake.output.offshore_shapes) diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index 180007b7..d285691a 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -27,9 +27,9 @@ if __name__ == "__main__": cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - config = snakemake.params["solar_thermal"] + config = snakemake.params.solar_thermal - time = pd.date_range(freq="h", **snakemake.params["snapshots"]) + time = pd.date_range(freq="h", **snakemake.params.snapshots) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) clustered_regions = ( diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index ee06aebb..9db37c25 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -27,7 +27,7 @@ if __name__ == "__main__": cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - time = pd.date_range(freq="h", **snakemake.params["snapshots"]) + time = pd.date_range(freq="h", **snakemake.params.snapshots) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) clustered_regions = ( diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py index dc2b94b9..c5bf4632 100644 --- a/scripts/build_transport_demand.py +++ b/scripts/build_transport_demand.py @@ -175,9 +175,9 @@ if __name__ == "__main__": snakemake.input.pop_weighted_energy_totals, index_col=0 ) - options = snakemake.params["sector"] + options = snakemake.params.sector - snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"], tz="UTC") + snapshots = pd.date_range(freq="h", **snakemake.params.snapshots, tz="UTC") nyears = len(snapshots) / 8760 diff --git a/scripts/make_summary.py b/scripts/make_summary.py index da0712d7..00fca2fd 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -198,7 +198,7 @@ def calculate_costs(n, label, costs): def calculate_cumulative_cost(): - planning_horizons = snakemake.params["scenario"]["planning_horizons"] + planning_horizons = snakemake.params.scenario["planning_horizons"] cumulative_cost = pd.DataFrame( index=df["costs"].sum().index, @@ -688,19 +688,19 @@ if __name__ == "__main__": (cluster, ll, opt + sector_opt, planning_horizon): "results/" + snakemake.params.RDIR + f"/postnetworks/elec_s{simpl}_{cluster}_l{ll}_{opt}_{sector_opt}_{planning_horizon}.nc" - for simpl in snakemake.params["scenario"]["simpl"] - for cluster in snakemake.params["scenario"]["clusters"] - for opt in snakemake.params["scenario"]["opts"] - for sector_opt in snakemake.params["scenario"]["sector_opts"] - for ll in snakemake.params["scenario"]["ll"] - for planning_horizon in snakemake.params["scenario"]["planning_horizons"] + for simpl in snakemake.params.scenario["simpl"] + for cluster in snakemake.params.scenario["clusters"] + for opt in snakemake.params.scenario["opts"] + for sector_opt in snakemake.params.scenario["sector_opts"] + for ll in snakemake.params.scenario["ll"] + for planning_horizon in snakemake.params.scenario["planning_horizons"] } - Nyears = len(pd.date_range(freq="h", **snakemake.params["snapshots"])) / 8760 + Nyears = len(pd.date_range(freq="h", **snakemake.params.snapshots)) / 8760 costs_db = prepare_costs( snakemake.input.costs, - snakemake.params["costs"], + snakemake.params.costs, Nyears, ) @@ -710,7 +710,7 @@ if __name__ == "__main__": to_csv(df) - if snakemake.params["foresight"] == "myopic": + if snakemake.params.foresight == "myopic": cumulative_cost = calculate_cumulative_cost() cumulative_cost.to_csv( "results/" + snakemake.params.RDIR + "/csvs/cumulative_cost.csv" diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 399f46e8..184d86f0 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -70,7 +70,7 @@ def plot_map( transmission=False, with_legend=True, ): - tech_colors = snakemake.params["plotting"]["tech_colors"] + tech_colors = snakemake.params.plotting["tech_colors"] n = network.copy() assign_location(n) @@ -116,9 +116,7 @@ def plot_map( costs = costs.stack() # .sort_index() # hack because impossible to drop buses... - eu_location = snakemake.params["plotting"].get( - "eu_node_location", dict(x=-5.5, y=46) - ) + eu_location = snakemake.params.plotting.get("eu_node_location", dict(x=-5.5, y=46)) n.buses.loc["EU gas", "x"] = eu_location["x"] n.buses.loc["EU gas", "y"] = eu_location["y"] @@ -315,7 +313,7 @@ def plot_h2_map(network, regions): h2_new = n.links[n.links.carrier == "H2 pipeline"] h2_retro = n.links[n.links.carrier == "H2 pipeline retrofitted"] - if snakemake.params["foresight"] == "myopic": + if snakemake.params.foresight == "myopic": # sum capacitiy for pipelines from different investment periods h2_new = group_pipes(h2_new) @@ -558,7 +556,7 @@ def plot_ch4_map(network): link_widths_used = max_usage / linewidth_factor link_widths_used[max_usage < line_lower_threshold] = 0.0 - tech_colors = snakemake.params["plotting"]["tech_colors"] + tech_colors = snakemake.params.plotting["tech_colors"] pipe_colors = { "gas pipeline": "#f08080", @@ -700,7 +698,7 @@ def plot_map_without(network): # hack because impossible to drop buses... if "EU gas" in n.buses.index: - eu_location = snakemake.params["plotting"].get( + eu_location = snakemake.params.plotting.get( "eu_node_location", dict(x=-5.5, y=46) ) n.buses.loc["EU gas", "x"] = eu_location["x"] @@ -876,7 +874,7 @@ def plot_series(network, carrier="AC", name="test"): stacked=True, linewidth=0.0, color=[ - snakemake.params["plotting"]["tech_colors"][i.replace(suffix, "")] + snakemake.params.plotting["tech_colors"][i.replace(suffix, "")] for i in new_columns ], ) @@ -937,7 +935,7 @@ if __name__ == "__main__": regions = gpd.read_file(snakemake.input.regions).set_index("name") - map_opts = snakemake.params["plotting"]["map"] + map_opts = snakemake.params.plotting["map"] if map_opts["boundaries"] is None: map_opts["boundaries"] = regions.total_bounds[[0, 2, 1, 3]] + [-1, 1, -1, 1] diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 36f75207..e7de5473 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -142,7 +142,7 @@ def plot_costs(): df = df.groupby(df.index.map(rename_techs)).sum() - to_drop = df.index[df.max(axis=1) < snakemake.params["plotting"]["costs_threshold"]] + to_drop = df.index[df.max(axis=1) < snakemake.params.plotting["costs_threshold"]] logger.info( f"Dropping technology with costs below {snakemake.params['plotting']['costs_threshold']} EUR billion per year" @@ -165,7 +165,7 @@ def plot_costs(): kind="bar", ax=ax, stacked=True, - color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index], + color=[snakemake.params.plotting["tech_colors"][i] for i in new_index], ) handles, labels = ax.get_legend_handles_labels() @@ -173,7 +173,7 @@ def plot_costs(): handles.reverse() labels.reverse() - ax.set_ylim([0, snakemake.params["plotting"]["costs_max"]]) + ax.set_ylim([0, snakemake.params.plotting["costs_max"]]) ax.set_ylabel("System Cost [EUR billion per year]") @@ -201,7 +201,7 @@ def plot_energy(): df = df.groupby(df.index.map(rename_techs)).sum() to_drop = df.index[ - df.abs().max(axis=1) < snakemake.params["plotting"]["energy_threshold"] + df.abs().max(axis=1) < snakemake.params.plotting["energy_threshold"] ] logger.info( @@ -227,7 +227,7 @@ def plot_energy(): kind="bar", ax=ax, stacked=True, - color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index], + color=[snakemake.params.plotting["tech_colors"][i] for i in new_index], ) handles, labels = ax.get_legend_handles_labels() @@ -237,8 +237,8 @@ def plot_energy(): ax.set_ylim( [ - snakemake.params["plotting"]["energy_min"], - snakemake.params["plotting"]["energy_max"], + snakemake.params.plotting["energy_min"], + snakemake.params.plotting["energy_max"], ] ) @@ -287,7 +287,7 @@ def plot_balances(): df = df.groupby(df.index.map(rename_techs)).sum() to_drop = df.index[ - df.abs().max(axis=1) < snakemake.params["plotting"]["energy_threshold"] / 10 + df.abs().max(axis=1) < snakemake.params.plotting["energy_threshold"] / 10 ] if v[0] in co2_carriers: @@ -317,7 +317,7 @@ def plot_balances(): kind="bar", ax=ax, stacked=True, - color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index], + color=[snakemake.params.plotting["tech_colors"][i] for i in new_index], ) handles, labels = ax.get_legend_handles_labels() @@ -455,10 +455,10 @@ def plot_carbon_budget_distribution(input_eurostat): ax1 = plt.subplot(gs1[0, 0]) ax1.set_ylabel("CO$_2$ emissions (Gt per year)", fontsize=22) ax1.set_ylim([0, 5]) - ax1.set_xlim([1990, snakemake.params["planning_horizons"][-1] + 1]) + ax1.set_xlim([1990, snakemake.params.planning_horizons[-1] + 1]) path_cb = "results/" + snakemake.params.RDIR + "/csvs/" - countries = snakemake.params["countries"] + countries = snakemake.params.countries e_1990 = co2_emissions_year(countries, input_eurostat, opts, year=1990) CO2_CAP = pd.read_csv(path_cb + "carbon_budget_distribution.csv", index_col=0) @@ -555,7 +555,7 @@ if __name__ == "__main__": plot_balances() - for sector_opts in snakemake.params["sector_opts"]: + for sector_opts in snakemake.params.sector_opts: opts = sector_opts.split("-") for o in opts: if "cb" in o: diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 51777ef5..e48ee162 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -253,12 +253,12 @@ if __name__ == "__main__": Nyears = n.snapshot_weightings.objective.sum() / 8760.0 costs = load_costs( snakemake.input.tech_costs, - snakemake.params["costs"], - snakemake.params["max_hours"], + snakemake.params.costs, + snakemake.params.max_hours, Nyears, ) - set_line_s_max_pu(n, snakemake.params["lines"]["s_max_pu"]) + set_line_s_max_pu(n, snakemake.params.lines["s_max_pu"]) for o in opts: m = re.match(r"^\d+h$", o, re.IGNORECASE) @@ -269,7 +269,7 @@ if __name__ == "__main__": for o in opts: m = re.match(r"^\d+seg$", o, re.IGNORECASE) if m is not None: - solver_name = snakemake.params["solver_name"] + solver_name = snakemake.params.solver_name n = apply_time_segmentation(n, m.group(0)[:-3], solver_name) break @@ -277,11 +277,11 @@ if __name__ == "__main__": if "Co2L" in o: m = re.findall("[0-9]*\.?[0-9]+$", o) if len(m) > 0: - co2limit = float(m[0]) * snakemake.params["co2base"] + co2limit = float(m[0]) * snakemake.params.co2base add_co2limit(n, co2limit, Nyears) logger.info("Setting CO2 limit according to wildcard value.") else: - add_co2limit(n, snakemake.params["co2limit"], Nyears) + add_co2limit(n, snakemake.params.co2limit, Nyears) logger.info("Setting CO2 limit according to config value.") break @@ -293,7 +293,7 @@ if __name__ == "__main__": add_gaslimit(n, limit, Nyears) logger.info("Setting gas usage limit according to wildcard value.") else: - add_gaslimit(n, snakemake.params["gaslimit"], Nyears) + add_gaslimit(n, snakemake.params.gaslimit, Nyears) logger.info("Setting gas usage limit according to config value.") break @@ -322,7 +322,7 @@ if __name__ == "__main__": add_emission_prices(n, dict(co2=float(m[0]))) else: logger.info("Setting emission prices according to config value.") - add_emission_prices(n, snakemake.params["costs"]["emission_prices"]) + add_emission_prices(n, snakemake.params.costs["emission_prices"]) break ll_type, factor = snakemake.wildcards.ll[0], snakemake.wildcards.ll[1:] @@ -330,8 +330,8 @@ if __name__ == "__main__": set_line_nom_max( n, - s_nom_max_set=snakemake.params["lines"].get("s_nom_max,", np.inf), - p_nom_max_set=snakemake.params["links"].get("p_nom_max,", np.inf), + s_nom_max_set=snakemake.params.lines.get("s_nom_max,", np.inf), + p_nom_max_set=snakemake.params.links.get("p_nom_max,", np.inf), ) if "ATK" in opts: diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 8b8b7d80..77fab4f8 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -200,12 +200,12 @@ def co2_emissions_year( """ Calculate CO2 emissions in one specific year (e.g. 1990 or 2018). """ - emissions_scope = snakemake.params["energy"]["emissions"] + emissions_scope = snakemake.params.energy["emissions"] eea_co2 = build_eea_co2(snakemake.input.co2, year, emissions_scope) # TODO: read Eurostat data from year > 2014 # this only affects the estimation of CO2 emissions for BA, RS, AL, ME, MK - report_year = snakemake.params["energy"]["eurostat_report_year"] + report_year = snakemake.params.energy["eurostat_report_year"] if year > 2014: eurostat_co2 = build_eurostat_co2( input_eurostat, countries, report_year, year=2014 @@ -241,7 +241,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): carbon_budget = float(o[o.find("cb") + 2 : o.find("ex")]) r = float(o[o.find("ex") + 2 :]) - countries = snakemake.params["countries"] + countries = snakemake.params.countries e_1990 = co2_emissions_year( countries, input_eurostat, opts, emissions_scope, report_year, year=1990 @@ -252,7 +252,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): countries, input_eurostat, opts, emissions_scope, report_year, year=2018 ) - planning_horizons = snakemake.params["planning_horizons"] + planning_horizons = snakemake.params.planning_horizons t_0 = planning_horizons[0] if "be" in o: @@ -391,7 +391,7 @@ def update_wind_solar_costs(n, costs): with xr.open_dataset(profile) as ds: underwater_fraction = ds["underwater_fraction"].to_pandas() connection_cost = ( - snakemake.params["length_factor"] + snakemake.params.length_factor * ds["average_distance"].to_pandas() * ( underwater_fraction @@ -483,8 +483,8 @@ def remove_elec_base_techs(n): batteries and H2) from base electricity-only network, since they're added here differently using links. """ - for c in n.iterate_components(snakemake.params["pypsa_eur"]): - to_keep = snakemake.params["pypsa_eur"][c.name] + for c in n.iterate_components(snakemake.params.pypsa_eur): + to_keep = snakemake.params.pypsa_eur[c.name] to_remove = pd.Index(c.df.carrier.unique()).symmetric_difference(to_keep) if to_remove.empty: continue @@ -674,7 +674,7 @@ def add_dac(n, costs): def add_co2limit(n, nyears=1.0, limit=0.0): logger.info(f"Adding CO2 budget limit as per unit of 1990 levels of {limit}") - countries = snakemake.params["countries"] + countries = snakemake.params.countries sectors = emission_sectors_from_opts(opts) @@ -787,7 +787,7 @@ def add_ammonia(n, costs): nodes = pop_layout.index - cf_industry = snakemake.params["industry"] + cf_industry = snakemake.params.industry n.add("Carrier", "NH3") @@ -1102,7 +1102,7 @@ def add_storage_and_grids(n, costs): lifetime=costs.at["OCGT", "lifetime"], ) - cavern_types = snakemake.params["sector"]["hydrogen_underground_storage_locations"] + cavern_types = snakemake.params.sector["hydrogen_underground_storage_locations"] h2_caverns = pd.read_csv(snakemake.input.h2_cavern, index_col=0) if ( @@ -3274,7 +3274,7 @@ if __name__ == "__main__": update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) - options = snakemake.params["sector"] + options = snakemake.params.sector opts = snakemake.wildcards.sector_opts.split("-") @@ -3289,7 +3289,7 @@ if __name__ == "__main__": costs = prepare_costs( snakemake.input.costs, - snakemake.params["costs"], + snakemake.params.costs, nyears, ) @@ -3301,10 +3301,10 @@ if __name__ == "__main__": spatial = define_spatial(pop_layout.index, options) - if snakemake.params["foresight"] == "myopic": + if snakemake.params.foresight == "myopic": add_lifetime_wind_solar(n, costs) - conventional = snakemake.params["conventional_carriers"] + conventional = snakemake.params.conventional_carriers for carrier in conventional: add_carrier_buses(n, carrier) @@ -3369,19 +3369,19 @@ if __name__ == "__main__": if options["allam_cycle"]: add_allam(n, costs) - solver_name = snakemake.params["solver_name"] + solver_name = snakemake.params.solver_name n = set_temporal_aggregation(n, opts, solver_name) limit_type = "config" - limit = get(snakemake.params["co2_budget"], investment_year) + limit = get(snakemake.params.co2_budget, investment_year) for o in opts: if "cb" not in o: continue limit_type = "carbon budget" fn = "results/" + snakemake.params.RDIR + "/csvs/carbon_budget_distribution.csv" if not os.path.exists(fn): - emissions_scope = snakemake.params["emissions_scope"] - report_year = snakemake.params["report_year"] + emissions_scope = snakemake.params.emissions_scope + report_year = snakemake.params.report_year build_carbon_budget( o, snakemake.input.eurostat, fn, emissions_scope, report_year ) @@ -3416,8 +3416,8 @@ if __name__ == "__main__": if options["electricity_grid_connection"]: add_electricity_grid_connection(n, costs) - first_year_myopic = (snakemake.params["foresight"] == "myopic") and ( - snakemake.params["planning_horizons"][0] == investment_year + first_year_myopic = (snakemake.params.foresight == "myopic") and ( + snakemake.params.planning_horizons[0] == investment_year ) if options.get("cluster_heat_buses", False) and not first_year_myopic: diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index 2204ac36..c7053bed 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -53,7 +53,7 @@ if __name__ == "__main__": snakemake ) # TODO Make logging compatible with progressbar (see PR #102) - if snakemake.params["tutorial"]: + if snakemake.params.tutorial: url = "https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz" else: url = "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz" diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 41679a03..d80e38a0 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -44,9 +44,9 @@ pypsa.pf.logger.setLevel(logging.WARNING) from pypsa.descriptors import get_switchable_as_dense as get_as_dense -def add_land_use_constraint(n, planning_horizons_param, config): +def add_land_use_constraint(n, planning_horizons, config): if "m" in snakemake.wildcards.clusters: - _add_land_use_constraint_m(n, planning_horizons_param, config) + _add_land_use_constraint_m(n, planning_horizons, config) else: _add_land_use_constraint(n) @@ -80,7 +80,7 @@ def _add_land_use_constraint(n): n.generators.p_nom_max.clip(lower=0, inplace=True) -def _add_land_use_constraint_m(n, planning_horizons_param, config): +def _add_land_use_constraint_m(n, planning_horizons, config): # if generators clustering is lower than network clustering, land_use accounting is at generators clusters planning_horizons = param["planning_horizons"] @@ -145,9 +145,9 @@ def prepare_network( n, solve_opts=None, config=None, - foresight_param=None, - planning_horizons_param=None, - co2_sequestration_potential_param=None, + foresight=None, + planning_horizons=None, + co2_sequestration_potential=None, ): if "clip_p_max_pu" in solve_opts: for df in ( @@ -198,11 +198,11 @@ def prepare_network( n.set_snapshots(n.snapshots[:nhours]) n.snapshot_weightings[:] = 8760.0 / nhours - if foresight_param == "myopic": - add_land_use_constraint(n, planning_horizons_param, config) + if foresight == "myopic": + add_land_use_constraint(n, planning_horizons, config) if n.stores.carrier.eq("co2 stored").any(): - limit = co2_sequestration_potential_param + limit = co2_sequestration_potential add_co2_sequestration_limit(n, limit=limit) return n @@ -597,13 +597,11 @@ def extra_functionality(n, snapshots): add_pipe_retrofit_constraint(n) -def solve_network(n, config, solving_param, opts="", **kwargs): - set_of_options = solving_param["solver"]["options"] - solver_options = ( - solving_param["solver_options"][set_of_options] if set_of_options else {} - ) - solver_name = solving_param["solver"]["name"] - cf_solving = solving_param["options"] +def solve_network(n, config, solving, opts="", **kwargs): + set_of_options = solving["solver"]["options"] + solver_options = solving["solver_options"][set_of_options] if set_of_options else {} + solver_name = solving["solver"]["name"] + cf_solving = solving["options"] track_iterations = cf_solving.get("track_iterations", False) min_iterations = cf_solving.get("min_iterations", 4) max_iterations = cf_solving.get("max_iterations", 6) @@ -672,7 +670,7 @@ if __name__ == "__main__": if "sector_opts" in snakemake.wildcards.keys(): opts += "-" + snakemake.wildcards.sector_opts opts = [o for o in opts.split("-") if o != ""] - solve_opts = snakemake.params["solving"]["options"] + solve_opts = snakemake.params.solving["options"] np.random.seed(solve_opts.get("seed", 123)) @@ -686,17 +684,15 @@ if __name__ == "__main__": n, solve_opts, config=snakemake.config, - foresight_param=snakemake.params["foresight"], - planning_horizons_param=snakemake.params["planning_horizons"], - co2_sequestration_potential_param=snakemake.params[ - "co2_sequestration_potential" - ], + foresight=snakemake.params.foresight, + planning_horizons=snakemake.params.planning_horizons, + co2_sequestration_potential=snakemake.params["co2_sequestration_potential"], ) n = solve_network( n, config=snakemake.config, - solving_param=snakemake.params["solving"], + solving=snakemake.params.solving, opts=opts, log_fn=snakemake.log.solver, ) diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 421bc515..37e853e5 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -41,7 +41,7 @@ if __name__ == "__main__": opts = (snakemake.wildcards.opts + "-" + snakemake.wildcards.sector_opts).split("-") opts = [o for o in opts if o != ""] - solve_opts = snakemake.params["options"] + solve_opts = snakemake.params.options np.random.seed(solve_opts.get("seed", 123)) From 27eea273bd20559dcf02d32ed0638c2938fb4acf Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 15 Jun 2023 19:12:30 +0200 Subject: [PATCH 39/41] harmonize params names --- rules/build_electricity.smk | 9 +++------ rules/build_sector.smk | 7 ++++--- scripts/add_electricity.py | 21 ++++++--------------- scripts/build_sequestration_potentials.py | 2 +- scripts/prepare_network.py | 2 +- scripts/prepare_sector_network.py | 4 ++-- 6 files changed, 17 insertions(+), 28 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 9d4315df..44ebf404 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -320,11 +320,10 @@ rule simplify_network: aggregation_strategies=config["clustering"].get("aggregation_strategies", {}), focus_weights=config.get("focus_weights", None), renewable_carriers=config["electricity"]["renewable_carriers"], - costs=config["costs"], max_hours=config["electricity"]["max_hours"], length_factor=config["lines"]["length_factor"], p_max_pu=config["links"].get("p_max_pu", 1.0), - solver_name=config["solving"]["solver"]["name"], + costs=config["costs"], input: network=RESOURCES + "networks/elec.nc", tech_costs=COSTS, @@ -357,10 +356,9 @@ rule cluster_network: focus_weights=config.get("focus_weights", None), renewable_carriers=config["electricity"]["renewable_carriers"], conventional_carriers=config["electricity"].get("conventional_carriers", []), - costs=config["costs"], max_hours=config["electricity"]["max_hours"], length_factor=config["lines"]["length_factor"], - solver_name=config["solving"]["solver"]["name"], + costs=config["costs"], input: network=RESOURCES + "networks/elec_s{simpl}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson", @@ -393,9 +391,9 @@ rule cluster_network: rule add_extra_components: params: - costs=config["costs"], extendable_carriers=config["electricity"]["extendable_carriers"], max_hours=config["electricity"]["max_hours"], + costs=config["costs"], input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", tech_costs=COSTS, @@ -418,7 +416,6 @@ rule prepare_network: params: links=config["links"], lines=config["lines"], - solver_name=config["solving"]["solver"]["name"], co2base=config["electricity"]["co2base"], co2limit=config["electricity"]["co2limit"], gaslimit=config["electricity"].get("gaslimit"), diff --git a/rules/build_sector.smk b/rules/build_sector.smk index e15394ff..1c3507fd 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -330,7 +330,9 @@ if config["sector"]["regional_co2_sequestration_potential"]["enable"]: rule build_sequestration_potentials: params: - co2seq_potential=config["sector"]["regional_co2_sequestration_potential"], + sequestration_potential=config["sector"][ + "regional_co2_sequestration_potential" + ], input: sequestration_potential=HTTP.remote( "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson", @@ -705,7 +707,6 @@ rule build_transport_demand: rule prepare_sector_network: params: co2_budget=config["co2_budget"], - solver_name=config["solving"]["solver"]["name"], conventional_carriers=config["existing_capacities"]["conventional_carriers"], foresight=config["foresight"], costs=config["costs"], @@ -716,7 +717,7 @@ rule prepare_sector_network: planning_horizons=config["scenario"]["planning_horizons"], countries=config["countries"], emissions_scope=config["energy"]["emissions"], - report_year=config["energy"]["eurostat_report_year"], + eurostat_report_year=config["energy"]["eurostat_report_year"], RDIR=RDIR, input: **build_retro_cost_output, diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index d98dc767..e2f7fb65 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -137,7 +137,7 @@ def _add_missing_carriers_from_costs(n, costs, carriers): n.import_components_from_dataframe(emissions, "Carrier") -def load_costs(tech_costs, params, max_hours, Nyears=1.0): +def load_costs(tech_costs, costs, max_hours, Nyears=1.0): # set all asset costs and other parameters costs = pd.read_csv(tech_costs, index_col=[0, 1]).sort_index() @@ -145,7 +145,7 @@ def load_costs(tech_costs, params, max_hours, Nyears=1.0): costs.loc[costs.unit.str.contains("/kW"), "value"] *= 1e3 costs.unit = costs.unit.str.replace("/kW", "/MW") - fill_values = params["fill_values"] + fill_values = costs["fill_values"] costs = costs.value.unstack().fillna(fill_values) costs["capital_cost"] = ( @@ -168,8 +168,8 @@ def load_costs(tech_costs, params, max_hours, Nyears=1.0): costs.at["CCGT", "co2_emissions"] = costs.at["gas", "co2_emissions"] costs.at["solar", "capital_cost"] = ( - params["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"] - + (1 - params["rooftop_share"]) * costs.at["solar-utility", "capital_cost"] + costs["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"] + + (1 - costs["rooftop_share"]) * costs.at["solar-utility", "capital_cost"] ) def costs_for_storage(store, link1, link2=None, max_hours=1.0): @@ -193,7 +193,7 @@ def load_costs(tech_costs, params, max_hours, Nyears=1.0): ) for attr in ("marginal_cost", "capital_cost"): - overwrites = params.get(attr) + overwrites = costs.get(attr) if overwrites is not None: overwrites = pd.Series(overwrites) costs.loc[overwrites.index, attr] = overwrites @@ -728,16 +728,6 @@ if __name__ == "__main__": ) ppl = load_powerplants(snakemake.input.powerplants) - if "renewable_carriers" in params.electricity: - renewable_carriers = set(params.electricity["renewable_carriers"]) - else: - logger.warning( - "Missing key `renewable_carriers` under config entry `electricity`. " - "In future versions, this will raise an error. " - "Falling back to carriers listed under `renewable`." - ) - renewable_carriers = params.renewable - attach_load( n, snakemake.input.regions, @@ -749,6 +739,7 @@ if __name__ == "__main__": update_transmission_costs(n, costs, params.length_factor) + renewable_carriers = set(params.electricity["renewable_carriers"]) extendable_carriers = params.electricity["extendable_carriers"] conventional_carriers = params.electricity["conventional_carriers"] conventional_inputs = { diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py index 0e59e55b..e19a96da 100644 --- a/scripts/build_sequestration_potentials.py +++ b/scripts/build_sequestration_potentials.py @@ -41,7 +41,7 @@ if __name__ == "__main__": "build_sequestration_potentials", simpl="", clusters="181" ) - cf = snakemake.params.co2seq_potential + cf = snakemake.params.sequestration_potential gdf = gpd.read_file(snakemake.input.sequestration_potential[0]) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index e48ee162..adbfa36c 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -269,7 +269,7 @@ if __name__ == "__main__": for o in opts: m = re.match(r"^\d+seg$", o, re.IGNORECASE) if m is not None: - solver_name = snakemake.params.solver_name + solver_name = snakemake.config["solving"]["solver"]["name"] n = apply_time_segmentation(n, m.group(0)[:-3], solver_name) break diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 77fab4f8..73adf522 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3369,7 +3369,7 @@ if __name__ == "__main__": if options["allam_cycle"]: add_allam(n, costs) - solver_name = snakemake.params.solver_name + solver_name = snakemake.config["solving"]["solver"]["name"] n = set_temporal_aggregation(n, opts, solver_name) limit_type = "config" @@ -3381,7 +3381,7 @@ if __name__ == "__main__": fn = "results/" + snakemake.params.RDIR + "/csvs/carbon_budget_distribution.csv" if not os.path.exists(fn): emissions_scope = snakemake.params.emissions_scope - report_year = snakemake.params.report_year + report_year = snakemake.params.eurostat_report_year build_carbon_budget( o, snakemake.input.eurostat, fn, emissions_scope, report_year ) From a17647a07e3c5e93fcaa2a686e0f72c09ab0dbfc Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 15 Jun 2023 19:35:41 +0200 Subject: [PATCH 40/41] add_electricity: fix cost ref --- scripts/add_electricity.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index e2f7fb65..594988bd 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -137,7 +137,7 @@ def _add_missing_carriers_from_costs(n, costs, carriers): n.import_components_from_dataframe(emissions, "Carrier") -def load_costs(tech_costs, costs, max_hours, Nyears=1.0): +def load_costs(tech_costs, config, max_hours, Nyears=1.0): # set all asset costs and other parameters costs = pd.read_csv(tech_costs, index_col=[0, 1]).sort_index() @@ -145,7 +145,7 @@ def load_costs(tech_costs, costs, max_hours, Nyears=1.0): costs.loc[costs.unit.str.contains("/kW"), "value"] *= 1e3 costs.unit = costs.unit.str.replace("/kW", "/MW") - fill_values = costs["fill_values"] + fill_values = config["fill_values"] costs = costs.value.unstack().fillna(fill_values) costs["capital_cost"] = ( @@ -168,8 +168,8 @@ def load_costs(tech_costs, costs, max_hours, Nyears=1.0): costs.at["CCGT", "co2_emissions"] = costs.at["gas", "co2_emissions"] costs.at["solar", "capital_cost"] = ( - costs["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"] - + (1 - costs["rooftop_share"]) * costs.at["solar-utility", "capital_cost"] + config["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"] + + (1 - config["rooftop_share"]) * costs.at["solar-utility", "capital_cost"] ) def costs_for_storage(store, link1, link2=None, max_hours=1.0): @@ -193,7 +193,7 @@ def load_costs(tech_costs, costs, max_hours, Nyears=1.0): ) for attr in ("marginal_cost", "capital_cost"): - overwrites = costs.get(attr) + overwrites = config.get(attr) if overwrites is not None: overwrites = pd.Series(overwrites) costs.loc[overwrites.index, attr] = overwrites From 284e3f8e70c4c8f44346bb519967d82256d5b3b3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 22:43:01 +0000 Subject: [PATCH 41/41] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/codespell-project/codespell: v2.2.4 → v2.2.5](https://github.com/codespell-project/codespell/compare/v2.2.4...v2.2.5) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 826a4819..5acfbf42 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,7 +30,7 @@ repos: # Find common spelling mistakes in comments and docstrings - repo: https://github.com/codespell-project/codespell - rev: v2.2.4 + rev: v2.2.5 hooks: - id: codespell args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom,appartment,bage,ore,setis,tabacco,berfore'] # Ignore capital case words, e.g. country codes