From 7bc9b8012c1fe1c747d6386656bade8be70d7a5e Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 4 Apr 2022 18:07:48 +0200 Subject: [PATCH 01/61] powerplants: update to ppm >= v0.5.1 --- envs/environment.yaml | 2 +- scripts/build_powerplants.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 3c69b77b..795aa334 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -24,7 +24,6 @@ dependencies: - yaml - pytables - lxml - - powerplantmatching>=0.4.8 - numpy - pandas - geopandas @@ -57,3 +56,4 @@ dependencies: - pip: - vresutils>=0.3.1 - tsam>=1.1.0 + - powerplantmatching>=0.5.1 diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index 764028d1..e18232b8 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -104,7 +104,7 @@ if __name__ == "__main__": countries = n.buses.country.unique() ppl = (pm.powerplants(from_url=True) - .powerplant.fill_missing_decommyears() + .powerplant.fill_missing_decommissioning_years() .powerplant.convert_country_to_alpha2() .query('Fueltype not in ["Solar", "Wind"] and Country in @countries') .replace({'Technology': {'Steam Turbine': 'OCGT'}}) From ade22bf4f003b774185e4a4ad123a6c33913bb0e Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 4 Apr 2022 19:03:09 +0200 Subject: [PATCH 02/61] add existing wind and solar capacities based on IRENASTATS --- config.default.yaml | 18 +++++++++++---- doc/release_notes.rst | 20 ++++++++++++++++ envs/environment.yaml | 2 +- scripts/add_electricity.py | 47 ++++++++++++++++++++------------------ 4 files changed, 60 insertions(+), 27 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index d2bf6159..f952f5b7 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -58,10 +58,20 @@ electricity: conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] renewable_capacities_from_OPSD: [] # onwind, offwind, solar - # estimate_renewable_capacities_from_capacity_stats: - # # Wind is the Fueltype in ppm.data.Capacity_stats, onwind, offwind-{ac,dc} the carrier in PyPSA-Eur - # Wind: [onwind, offwind-ac, offwind-dc] - # Solar: [solar] + estimate_renewable_capacities: + # Renewable capacities are based on existing capacities reported by IRENA + + # Reference year, any of 2000 to 2020 + year: 2020 + # Artificially limit maximum capacities to factor * (IRENA capacities), + # i.e. 110% of 's capacities => expansion_limit: 1.1 + # false: Use estimated renewable potentials determine by the workflow + expansion_limit: false + technology_mapping: + # Wind is the Fueltype in ppm.data.Capacity_stats, onwind, offwind-{ac,dc} the carrier in PyPSA-Eur + Offshore: [offwind-ac, offwind-dc] + Onshore: [onwind] + PV: [solar] atlite: nprocesses: 4 diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 3f131dc0..9b012bf8 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -7,6 +7,26 @@ Release Notes ########################################## +Energy Security Release (April 2022) +==================================== + +**New Features and Changes** + +* Added existing renewable capacities for all countries based on IRENA statistics (IRENASTAT) using new ``powerplantmatching`` version: + * Configuration of reference year for capacities can be configured (default: ``2020``) + * Uniform expansion limit of renewable build-up based on existing capacities can be configured using ``expansion_limit`` option + (default: ``false``; limited to determined renewable potentials) + * Distribution of country-level capacities proportional to maximum annual energy yield for each bus region + * This functionality was previously using OPSD data. + * The corresponding ``config`` entries changed, cf. ``config.default.yaml``: + * old: ``estimate_renewable_capacities_from_capacity_stats`` + * new: ``estimate_renewable_capacities`` + + +**Bugs and Compatibility** + +* ``powerplantmatching>=0.5.1`` is now required for ``IRENASTATS``. + Synchronisation Release - Ukraine and Moldova (17th March 2022) =============================================================== diff --git a/envs/environment.yaml b/envs/environment.yaml index 3c69b77b..d02b0018 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -24,7 +24,7 @@ dependencies: - yaml - pytables - lxml - - powerplantmatching>=0.4.8 + - powerplantmatching>=0.5.1 - numpy - pandas - geopandas diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index ad932cd8..f468bab3 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: : 2017-2020 The PyPSA-Eur Authors +# SPDX-FileCopyrightText: : 2017-2022 The PyPSA-Eur Authors # # SPDX-License-Identifier: MIT @@ -491,29 +491,29 @@ def attach_OPSD_renewables(n, techs): -def estimate_renewable_capacities(n, tech_map): +def estimate_renewable_capacities(n, config): - if len(tech_map) == 0: return - - capacities = (pm.data.Capacity_stats().powerplant.convert_country_to_alpha2() - [lambda df: df.Energy_Source_Level_2] - .set_index(['Fueltype', 'Country']).sort_index()) - - countries = n.buses.country.unique() + if not config["electricity"]["estimate_renewable_capacities"]: return + + year = config["electricity"]["estimate_renewable_capacities"]["year"] + tech_map = config["electricity"]["estimate_renewable_capacities"]["technology_mapping"] + tech_keys = list(tech_map.keys()) + countries = config["countries"] + expansion_limit = config["electricity"]["estimate_renewable_capacities"]["expansion_limit"] if len(countries) == 0: return + if len(tech_map) == 0: return - logger.info('heuristics applied to distribute renewable capacities [MW] \n{}' - .format(capacities.query('Fueltype in @tech_map.keys() and Capacity >= 0.1') - .groupby('Country').agg({'Capacity': 'sum'}))) + capacities = pm.data.IRENASTAT().powerplant.convert_country_to_alpha2() + capacities = capacities.query("Year == @year and Technology in @tech_keys and Country in @countries") + capacities = capacities.groupby(["Technology", "Country"]).Capacity.sum() - for ppm_fueltype, techs in tech_map.items(): - tech_capacities = capacities.loc[ppm_fueltype, 'Capacity']\ - .reindex(countries, fill_value=0.) - #tech_i = n.generators.query('carrier in @techs').index - tech_i = (n.generators.query('carrier in @techs') - [n.generators.query('carrier in @techs') - .bus.map(n.buses.country).isin(countries)].index) + logger.info(f"Heuristics applied to distribute renewable capacities [MW] " + f"{capacities.groupby('Country').sum()}") + + for ppm_technology, techs in tech_map.items(): + tech_capacities = capacities.loc[ppm_technology].reindex(countries, fill_value=0.) + tech_i = n.generators.query('carrier in @techs').index n.generators.loc[tech_i, 'p_nom'] = ( (n.generators_t.p_max_pu[tech_i].mean() * n.generators.loc[tech_i, 'p_nom_max']) # maximal yearly generation @@ -522,6 +522,11 @@ def estimate_renewable_capacities(n, tech_map): .where(lambda s: s>0.1, 0.)) # only capacities above 100kW n.generators.loc[tech_i, 'p_nom_min'] = n.generators.loc[tech_i, 'p_nom'] + if expansion_limit: + assert np.isscalar(expansion_limit) + logger.info(f"Reducing capacity expansion limit to {expansion_limit*100:.2f}% of installed capacity.") + n.generators.loc[tech_i, 'p_nom_max'] = float(expansion_limit) * n.generators.loc[tech_i, 'p_nom_min'] + def add_nice_carrier_names(n, config): carrier_i = n.carriers.index @@ -565,11 +570,9 @@ if __name__ == "__main__": carriers = snakemake.config['electricity']['extendable_carriers']['Generator'] attach_extendable_generators(n, costs, ppl, carriers) - tech_map = snakemake.config['electricity'].get('estimate_renewable_capacities_from_capacity_stats', {}) - estimate_renewable_capacities(n, tech_map) + estimate_renewable_capacities(n, snakemake.config) techs = snakemake.config['electricity'].get('renewable_capacities_from_OPSD', []) attach_OPSD_renewables(n, techs) - update_p_nom_max(n) add_nice_carrier_names(n, snakemake.config) From f878faac73b9ef61d0a4c146c8ac6be0a990dd7b Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 5 Apr 2022 08:02:44 +0200 Subject: [PATCH 03/61] add_electricity: allow estimate_renewable_capacities to be commented out --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index f468bab3..961b87dc 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -493,7 +493,7 @@ def attach_OPSD_renewables(n, techs): def estimate_renewable_capacities(n, config): - if not config["electricity"]["estimate_renewable_capacities"]: return + if not config["electricity"].get("estimate_renewable_capacities"): return year = config["electricity"]["estimate_renewable_capacities"]["year"] tech_map = config["electricity"]["estimate_renewable_capacities"]["technology_mapping"] From 1fd6a685ab636ca918eb58a5e3dbac175d66ad68 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 5 Apr 2022 15:12:01 +0200 Subject: [PATCH 04/61] powerplants: filter out powerplants with shut down date < 2021 --- config.default.yaml | 6 ++++-- doc/release_notes.rst | 1 + scripts/build_powerplants.py | 13 +++++++------ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index f952f5b7..4cd0eadc 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -53,8 +53,10 @@ electricity: battery: 6 H2: 168 - powerplants_filter: false # use pandas query strings here, e.g. Country not in ['Germany'] - custom_powerplants: false # use pandas query strings here, e.g. Country in ['Germany'] + # use pandas query strings here, e.g. Country not in ['Germany'] + powerplants_filter: (DateOut >= 2021 or DateOut != DateOut) + # use pandas query strings here, e.g. Country in ['Germany'] + custom_powerplants: false conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] renewable_capacities_from_OPSD: [] # onwind, offwind, solar diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 9b012bf8..4ef5e956 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -22,6 +22,7 @@ Energy Security Release (April 2022) * old: ``estimate_renewable_capacities_from_capacity_stats`` * new: ``estimate_renewable_capacities`` +* The powerplants that have been shut down before 2021 are filtered out. **Bugs and Compatibility** diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index e18232b8..9a7c9e23 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -94,6 +94,10 @@ def add_custom_powerplants(ppl, custom_powerplants, custom_ppl_query=False): return pd.concat([ppl, add_ppls], sort=False, ignore_index=True, verify_integrity=True) +def replace_natural_gas_by_technology(df): + return df.Fueltype.where(df.Fueltype != 'Natural Gas', df.Technology) + + if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake @@ -103,16 +107,13 @@ if __name__ == "__main__": n = pypsa.Network(snakemake.input.base_network) countries = n.buses.country.unique() + ppl = (pm.powerplants(from_url=True) .powerplant.fill_missing_decommissioning_years() .powerplant.convert_country_to_alpha2() .query('Fueltype not in ["Solar", "Wind"] and Country in @countries') - .replace({'Technology': {'Steam Turbine': 'OCGT'}}) - .assign(Fueltype=lambda df: ( - df.Fueltype - .where(df.Fueltype != 'Natural Gas', - df.Technology.replace('Steam Turbine', - 'OCGT').fillna('OCGT'))))) + .replace({'Technology': {'Steam Turbine': 'OCGT', "Combustion Engine": "OCGT"}}) + .assign(Fueltype=replace_natural_gas_by_technology)) ppl_query = snakemake.config['electricity']['powerplants_filter'] if isinstance(ppl_query, str): From 7a52b6bc455e160b9b32ca95400e5b882da32e0e Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 28 Mar 2022 12:02:08 +0200 Subject: [PATCH 05/61] resolve cherry merge conflict 1 --- Snakefile | 4 ++-- scripts/build_hydro_profile.py | 29 ++++++++++++++++++++++++++--- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/Snakefile b/Snakefile index 7678a401..f3a894fc 100644 --- a/Snakefile +++ b/Snakefile @@ -50,7 +50,7 @@ if config['enable'].get('prepare_links_p_nom', False): datafiles = ['ch_cantons.csv', 'je-e-21.03.02.xls', - 'eez/World_EEZ_v8_2014.shp', 'EIA_hydro_generation_2000_2014.csv', + 'eez/World_EEZ_v8_2014.shp', 'hydro_capacities.csv', 'naturalearth/ne_10m_admin_0_countries.shp', 'NUTS_2013_60M_SH/data/NUTS_RG_60M_2013.shp', 'nama_10r_3popgdp.tsv.gz', 'nama_10r_3gdp.tsv.gz', 'corine/g250_clc06_V18_5.tif'] @@ -208,7 +208,7 @@ rule build_renewable_profiles: rule build_hydro_profile: input: country_shapes='resources/country_shapes.geojson', - eia_hydro_generation='data/bundle/EIA_hydro_generation_2000_2014.csv', + eia_hydro_generation='data/eia_hydro_annual_generation.csv', cutout=f"cutouts/{config['renewable']['hydro']['cutout']}.nc" if "hydro" in config["renewable"] else "config['renewable']['hydro']['cutout'] not configured", output: 'resources/profile_hydro.nc' log: "logs/build_hydro_profile.log" diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 74efc2ef..0fb20b06 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -64,7 +64,29 @@ from _helpers import configure_logging import atlite import geopandas as gpd -from vresutils import hydro as vhydro +import pandas as pd + +import country_converter as coco +cc = coco.CountryConverter() + + +def get_eia_annual_hydro_generation(fn, countries): + + # in billion kWh/a = TWh/a + df = pd.read_csv(fn, skiprows=2, index_col=1, na_values=[u' ','--']).iloc[1:, 1:] + df.index = df.index.str.strip() + + df.loc["Germany"] = df.filter(like='Germany', axis=0).sum() + df.loc["Serbia"] += df.loc["Kosovo"] + df = df.loc[~df.index.str.contains('Former')] + + df.index = cc.convert(df.index, to='iso2') + df.index.name = 'countries' + + df = df.T[countries] * 1e6 # in MWh/a + + return df + logger = logging.getLogger(__name__) @@ -82,8 +104,9 @@ if __name__ == "__main__": .set_index('name')['geometry'].reindex(countries)) country_shapes.index.name = 'countries' - eia_stats = vhydro.get_eia_annual_hydro_generation( - snakemake.input.eia_hydro_generation).reindex(columns=countries) + fn = snakemake.input.eia_hydro_generation + eia_stats = get_eia_annual_hydro_generation(fn, countries) + inflow = cutout.runoff(shapes=country_shapes, smooth=True, lower_threshold_quantile=True, From 998761ec8851c3fae06c8e9c738cba7d31f80ed5 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 28 Mar 2022 12:03:23 +0200 Subject: [PATCH 06/61] build_hydro: add new EIA hydro dataset --- data/eia_hydro_annual_generation.csv | 50 ++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 data/eia_hydro_annual_generation.csv diff --git a/data/eia_hydro_annual_generation.csv b/data/eia_hydro_annual_generation.csv new file mode 100644 index 00000000..cb1ae12f --- /dev/null +++ b/data/eia_hydro_annual_generation.csv @@ -0,0 +1,50 @@ +https://www.eia.gov/international/data/world/electricity/electricity-generation?pd=2&p=000000000000000000000000000000g&u=1&f=A&v=mapbubble&a=-&i=none&vo=value&t=R&g=000000000000002&l=73-1028i008017kg6368g80a4k000e0ag00gg0004g8g0ho00g000400008&s=315532800000&e=1577836800000&ev=false& +Report generated on: 03-28-2022 11:20:48 +"API","","1980","1981","1982","1983","1984","1985","1986","1987","1988","1989","1990","1991","1992","1993","1994","1995","1996","1997","1998","1999","2000","2001","2002","2003","2004","2005","2006","2007","2008","2009","2010","2011","2012","2013","2014","2015","2016","2017","2018","2019","2020" +"","hydroelectricity net generation (billion kWh)","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","" +"INTL.33-12-EURO-BKWH.A"," Europe","458.018","464.155","459.881","473.685","481.241","476.739","459.535","491.085","534.517","465.365","474.466","475.47","509.041","526.448","531.815","543.743","529.114164","543.845616","562.441501","569.308453","591.206662","587.371195","541.542535","506.19703","544.536443","545.176179","537.335934","540.934407","567.557921","564.244482","619.96477","543.05273","600.46622","631.86431","619.59229","615.53013","629.98906","562.59258","619.31106","610.62616","670.925" +"INTL.33-12-ALB-BKWH.A"," Albania","2.919","3.018","3.093","3.167","3.241","3.315","3.365","3.979","3.713","3.846","2.82","3.483","3.187","3.281","3.733","4.162","5.669","4.978","4.872","5.231","4.548","3.519","3.477","5.117","5.411","5.319","4.951","2.76","3.759","5.201","7.49133","4.09068","4.67775","6.88941","4.67676","5.83605","7.70418","4.47975","8.46648","5.15394","5.281" +"INTL.33-12-AUT-BKWH.A"," Austria","28.501","30.008","29.893","29.577","28.384","30.288","30.496","25.401","35.151","34.641","31.179","31.112","34.483","36.336","35.349","36.696","33.874","35.744","36.792","40.292","41.418","40.05","39.825","32.883","36.394","36.31","35.48","36.732","37.969","40.487","36.466","32.511","41.862","40.138","39.001","35.255","37.954","36.462","35.73","40.43655","45.344" +"INTL.33-12-BEL-BKWH.A"," Belgium","0.274","0.377","0.325","0.331","0.348","0.282","0.339","0.425","0.354","0.3","0.263","0.226","0.338","0.252","0.342","0.335","0.237","0.30195","0.38511","0.338","0.455","0.437","0.356","0.245","0.314","0.285","0.355","0.385","0.406","0.325","0.298","0.193","0.353","0.376","0.289","0.314","0.367","0.268","0.311","0.108","1.29" +"INTL.33-12-BIH-BKWH.A"," Bosnia and Herzegovina","--","--","--","--","--","--","--","--","--","--","--","--","3.374","2.343","3.424","3.607","5.104","4.608","4.511","5.477","5.043","5.129","5.215","4.456","5.919","5.938","5.798","3.961","4.818","6.177","7.946","4.343","4.173","7.164","5.876","5.495","5.585","3.7521","6.35382","6.02019","6.1" +"INTL.33-12-BGR-BKWH.A"," Bulgaria","3.674","3.58","3.018","3.318","3.226","2.214","2.302","2.512","2.569","2.662","1.859","2.417","2.042","1.923","1.453","2.291","2.89","2.726","3.066","2.725","2.646","1.72","2.172","2.999","3.136","4.294","4.196","2.845","2.796","3.435","4.98168","2.84328","3.14622","3.99564","4.55598","5.59845","3.8412","2.79972","5.09553","3.34917","3.37" +"INTL.33-12-HRV-BKWH.A"," Croatia","--","--","--","--","--","--","--","--","--","--","--","--","4.298","4.302","4.881","5.212","7.156","5.234","5.403","6.524","5.794","6.482","5.311","4.827","6.888","6.27","5.94","4.194","5.164","6.663","9.035","4.983","4.789","8.536","8.917","6.327","6.784","5.255","7.62399","5.87268","3.4" +"INTL.33-12-CYP-BKWH.A"," Cyprus","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0" +"INTL.33-12-CZE-BKWH.A"," Czech Republic","--","--","--","--","--","--","--","--","--","--","--","--","--","1.355","1.445","1.982","1.949","1.68201","1.382","1.664","1.7404","2.033","2.467","1.369","1.999","2.356","2.525","2.068","2.004","2.405","2.775","1.95","2.107","2.704","1.909","1.779","1.983","1.852","1.615","1.98792","3.4" +"INTL.33-12-DNK-BKWH.A"," Denmark","0.03","0.031","0.028","0.036","0.028","0.027","0.029","0.029","0.032","0.027","0.027","0.026","0.028","0.027","0.033","0.03","0.019","0.019","0.02673","0.031","0.03","0.028","0.032","0.021","0.027","0.023","0.023","0.028","0.026","0.019","0.021","0.017","0.017","0.013","0.015","0.018","0.019","0.018","0.015","0.01584","0.02" +"INTL.33-12-EST-BKWH.A"," Estonia","--","--","--","--","--","--","--","--","--","--","--","--","0.001","0.001","0.003","0.002","0.002","0.003","0.004","0.004","0.005","0.007","0.006","0.013","0.022","0.022","0.014","0.021","0.028","0.032","0.027","0.03","0.042","0.026","0.027","0.027","0.035","0.026","0.015","0.01881","0.04" +"INTL.33-12-FRO-BKWH.A"," Faroe Islands","0.049","0.049","0.049","0.049","0.049","0.049","0.049","0.049","0.062","0.071","0.074","0.074","0.083","0.073","0.075","0.075","0.069564","0.075066","0.076501","0.069453","0.075262","0.075195","0.095535","0.08483","0.093443","0.097986","0.099934","0.103407","0.094921","0.091482","0.06676","0.092","0.099","0.091","0.121","0.132","0.105","0.11","0.107","0.102","0.11" +"INTL.33-12-FIN-BKWH.A"," Finland","10.115","13.518","12.958","13.445","13.115","12.211","12.266","13.658","13.229","12.9","10.75","13.065","14.956","13.341","11.669","12.796","11.742","12.11958","14.9","12.652","14.513","13.073","10.668","9.495","14.919","13.646","11.379","14.035","16.941","12.559","12.743","12.278","16.667","12.672","13.24","16.584","15.634","14.61","13.137","12.31461","15.56" +"INTL.33-12-CSK-BKWH.A"," Former Czechoslovakia","4.8","4.2","3.7","3.9","3.2","4.3","4","4.853","4.355","4.229","3.919","3.119","3.602","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-12-SCG-BKWH.A"," Former Serbia and Montenegro","--","--","--","--","--","--","--","--","--","--","--","--","11.23","10.395","11.016","12.071","14.266","12.636","12.763","13.243","11.88","12.326","11.633","9.752","11.01","11.912","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-12-YUG-BKWH.A"," Former Yugoslavia","27.868","25.044","23.295","21.623","25.645","24.363","27.474","25.98","25.612","23.256","19.601","18.929","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-12-FRA-BKWH.A"," France","68.253","70.358","68.6","67.515","64.01","60.248","60.953","68.623","73.952","45.744","52.796","56.277","68.313","64.3","78.057","72.196","64.43","63.151","61.479","71.832","66.466","73.888","59.992","58.567","59.276","50.965","55.741","57.029","63.017","56.428","61.945","45.184","59.099","71.042","62.993","54.876","60.094","49.389","64.485","56.98242","64.84" +"INTL.33-12-DEU-BKWH.A"," Germany","--","--","--","--","--","--","--","--","--","--","--","14.742","17.223","17.699","19.731","21.562","21.737","17.18343","17.044","19.451","21.515","22.506","22.893","19.071","20.866","19.442","19.808","20.957","20.239","18.841","20.678","17.323","21.331","22.66","19.31","18.664","20.214","19.985","17.815","19.86039","24.75" +"INTL.33-12-DDR-BKWH.A"," Germany, East","1.658","1.718","1.748","1.683","1.748","1.758","1.767","1.726","1.719","1.551","1.389","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-12-DEUW-BKWH.A"," Germany, West","17.125","17.889","17.694","16.713","16.434","15.354","16.526","18.36","18.128","16.482","15.769","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-12-GIB-BKWH.A"," Gibraltar","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0" +"INTL.33-12-GRC-BKWH.A"," Greece","3.396","3.398","3.551","2.331","2.852","2.792","3.222","2.768","2.354","1.888","1.751","3.068","2.181","2.26","2.573","3.494","4.305","3.84318","3.68","4.546","3.656","2.076","2.772","4.718","4.625","4.967","5.806","2.565","3.279","5.32","7.431","3.998","4.387","6.337","4.464","5.782","5.543","3.962","5.035","3.9798","3.43" +"INTL.33-12-HUN-BKWH.A"," Hungary","0.111","0.166","0.158","0.153","0.179","0.153","0.152","0.167","0.167","0.156","0.176","0.192","0.156","0.164","0.159","0.161","0.205","0.21384","0.15345","0.179","0.176","0.184","0.192","0.169","0.203","0.2","0.184","0.208","0.211","0.226","0.184","0.216","0.206","0.208","0.294","0.227","0.253","0.214","0.216","0.21681","0.24" +"INTL.33-12-ISL-BKWH.A"," Iceland","3.053","3.085","3.407","3.588","3.738","3.667","3.846","3.918","4.169","4.217","4.162","4.162","4.267","4.421","4.47","4.635","4.724","5.15493","5.565","5.987","6.292","6.512","6.907","7.017","7.063","6.949","7.22","8.31","12.303","12.156","12.51","12.382","12.214","12.747","12.554","13.541","13.092","13.892","13.679","13.32441","12.46" +"INTL.33-12-IRL-BKWH.A"," Ireland","0.833","0.855","0.792","0.776","0.68","0.824","0.91","0.673","0.862","0.684","0.69","0.738","0.809","0.757","0.911","0.706","0.715","0.67122","0.907","0.838","0.838","0.59","0.903","0.592","0.624","0.625","0.717","0.66","0.959","0.893","0.593","0.699","0.795","0.593","0.701","0.798","0.674","0.685","0.687","0.87813","1.21" +"INTL.33-12-ITA-BKWH.A"," Italy","44.997","42.782","41.216","40.96","41.923","40.616","40.626","39.05","40.205","33.647","31.31","41.817","41.778","41.011","44.212","37.404","41.617","41.18697","40.808","44.911","43.763","46.343","39.125","33.303","41.915","35.706","36.624","32.488","41.207","48.647","50.506","45.36477","41.45625","52.24626","57.95955","45.08163","42.00768","35.83701","48.29913","45.31824","47.72" +"INTL.33-12-XKS-BKWH.A"," Kosovo","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","0.075","0.119","0.154","0.104","0.095","0.142","0.149","0.139","0.243","0.177","0.27027","0.2079","0.26" +"INTL.33-12-LVA-BKWH.A"," Latvia","--","--","--","--","--","--","--","--","--","--","--","--","2.498","2.846","3.272","2.908","1.841","2.922","2.99","2.729","2.791","2.805","2.438","2.243","3.078","3.293","2.671","2.706","3.078","3.422","3.488","2.857","3.677","2.838","1.953","1.841","2.523","4.356","2.417","2.08692","2.59" +"INTL.33-12-LTU-BKWH.A"," Lithuania","--","--","--","--","--","--","--","--","--","--","--","--","0.308","0.389","0.447","0.369","0.323","0.291","0.413","0.409","0.336","0.322","0.35","0.323","0.417","0.446193","0.393","0.417","0.398","0.42","0.535","0.475","0.419","0.516","0.395","0.346","0.45","0.597","0.427","0.34254","1.06" +"INTL.33-12-LUX-BKWH.A"," Luxembourg","0.086","0.095","0.084","0.083","0.088","0.071","0.084","0.101","0.097","0.072","0.07","0.083","0.069","0.066","0.117","0.087","0.059","0.082","0.114","0.084","0.119","0.117","0.098","0.078","0.103","0.093","0.11","0.116","0.131","0.105","0.104","0.061","0.095","0.114","0.104","0.095","0.111","0.082","0.089","0.10593","1.09" +"INTL.33-12-MLT-BKWH.A"," Malta","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0" +"INTL.33-12-MNE-BKWH.A"," Montenegro","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","1.733","1.271","1.524","2.05","2.723","1.192","1.462","2.479","1.734","1.476","1.825","1.014","2.09187","1.78","1.8" +"INTL.33-12-NLD-BKWH.A"," Netherlands","0","0","0","0","0","0.003","0.003","0.001","0.002","0.037","0.119","0.079","0.119","0.091","0.1","0.087","0.079","0.09108","0.111","0.089","0.141","0.116","0.109","0.071","0.094","0.087","0.105","0.106","0.101","0.097","0.105","0.057","0.104","0.114","0.112","0.093","0.1","0.061","0.072","0.07326","0.05" +"INTL.33-12-MKD-BKWH.A"," North Macedonia","--","--","--","--","--","--","--","--","--","--","--","--","0.817","0.517","0.696","0.793","0.842","0.891","1.072","1.375","1.158","0.62","0.749","1.36","1.467","1.477","1.634","1","0.832","1.257","2.407","1.419","1.031","1.568","1.195","1.846","1.878","1.099","1.773","1.15236","1.24" +"INTL.33-12-NOR-BKWH.A"," Norway","82.717","91.876","91.507","104.704","104.895","101.464","95.321","102.341","107.919","117.369","119.933","109.032","115.505","118.024","110.398","120.315","102.823","108.677","114.546","120.237","140.4","119.258","128.078","104.425","107.693","134.331","118.175","132.319","137.654","124.03","116.257","119.78","141.189","127.551","134.844","136.662","142.244","141.651","138.202","123.66288","141.69" +"INTL.33-12-POL-BKWH.A"," Poland","2.326","2.116","1.528","1.658","1.394","1.833","1.534","1.644","1.775","1.593","1.403","1.411","1.492","1.473","1.716","1.868","1.912","1.941","2.286","2.133","2.085","2.302","2.256","1.654","2.06","2.179","2.022","2.328","2.13","2.351","2.9","2.313","2.02","2.421","2.165","1.814","2.117","2.552","1.949","1.93842","2.93" +"INTL.33-12-PRT-BKWH.A"," Portugal","7.873","4.934","6.82","7.897","9.609","10.512","8.364","9.005","12.037","5.72","9.065","8.952","4.599","8.453","10.551","8.26","14.613","12.97395","12.853","7.213","11.21","13.894","7.722","15.566","9.77","4.684","10.892","9.991","6.73","8.201","15.954","11.423","5.589","13.652","15.471","8.615","15.608","5.79","12.316","8.6526","13.96" +"INTL.33-12-ROU-BKWH.A"," Romania","12.506","12.605","11.731","9.934","11.208","11.772","10.688","11.084","13.479","12.497","10.87","14.107","11.583","12.64","12.916","16.526","15.597","17.334","18.69","18.107","14.63","14.774","15.886","13.126","16.348","20.005","18.172","15.806","17.023","15.379","19.684","14.581","11.945","14.807","18.618","16.467","17.848","14.349","17.48736","15.65289","15.53" +"INTL.33-12-SRB-BKWH.A"," Serbia","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","10.855","9.937","9.468","10.436","11.772","8.58","9.193","10.101","10.893","9.979","10.684","9.061","10.53261","10.07028","9.66" +"INTL.33-12-SVK-BKWH.A"," Slovakia","--","--","--","--","--","--","--","--","--","--","--","--","--","3.432","4.311","4.831","4.185","4.023","4.224","4.429","4.569","4.878","5.215","3.4452","4.059","4.592","4.355","4.406","4","4.324","5.184","3.211","3.687","4.329","3.762","3.701","4.302","4.321","3.506","4.27383","4.67" +"INTL.33-12-SVN-BKWH.A"," Slovenia","--","--","--","--","--","--","--","--","--","--","--","--","3.379","2.974","3.348","3.187","3.616","3.046","3.4","3.684","3.771","3.741","3.265","2.916","4.033","3.426","3.555","3.233","3.978","4.666","4.452","3.506","3.841","4.562","6.011","3.75","4.443","3.814","4.643","4.43421","5.24" +"INTL.33-12-ESP-BKWH.A"," Spain","29.16","21.64","25.99","26.696","31.088","30.895","26.105","27.016","34.76","19.046","25.16","27.01","18.731","24.133","27.898","22.881","39.404","34.43","33.665","22.634","29.274","40.617","22.691","40.643","31.359","18.209","25.699","27.036","23.13","26.147","41.576","30.07","20.192","36.45","38.815","27.656","35.77","18.007","33.743","24.23025","33.34" +"INTL.33-12-SWE-BKWH.A"," Sweden","58.133","59.006","54.369","62.801","67.106","70.095","60.134","70.95","69.016","70.911","71.778","62.603","73.588","73.905","58.508","67.421","51.2226","68.365","74.25","70.974","77.798","78.269","65.696","53.005","59.522","72.075","61.106","65.497","68.378","65.193","66.279","66.047","78.333","60.81","63.227","74.734","61.645","64.651","61.79","64.46583","71.6" +"INTL.33-12-CHE-BKWH.A"," Switzerland","32.481","35.13","35.974","35.069","29.871","31.731","32.576","34.328","35.437","29.477","29.497","31.756","32.373","35.416","38.678","34.817","28.458","33.70257","33.136","39.604","36.466","40.895","34.862","34.471","33.411","30.914","30.649","34.898","35.676","35.366","35.704","32.069","38.218","38.08","37.659","37.879","34.281","33.754","34.637","37.6596","40.62" +"INTL.33-12-TUR-BKWH.A"," Turkey","11.159","12.308","13.81","11.13","13.19","11.822","11.637","18.314","28.447","17.61","22.917","22.456","26.302","33.611","30.28","35.186","40.07","39.41784","41.80671","34.33","30.57","23.77","33.346","34.977","45.623","39.165","43.802","35.492","32.937","35.598","51.423","51.155","56.669","58.225","39.75","65.856","66.686","57.824","59.49","87.99714","77.39" +"INTL.33-12-GBR-BKWH.A"," United Kingdom","3.921","4.369","4.543","4.548","3.992","4.08","4.767","4.13","4.915","4.732","5.119","4.534","5.329","4.237","5.043","4.79","3.359","4.127","5.067","5.283","5.035","4.015","4.74","3.195","4.795","4.873","4.547","5.026","5.094","5.178","3.566","5.655","5.286","4.667","5.832","6.246","5.342","5.836","5.189","5.89941","7.64" \ No newline at end of file From 9812e64e8228be3e297fcf5e959e2bf31f526d35 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 29 Mar 2022 09:17:28 +0200 Subject: [PATCH 07/61] resolve cherry merge conflict 2 --- envs/environment.yaml | 2 +- scripts/build_hydro_profile.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 795aa334..9772d882 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -37,6 +37,7 @@ dependencies: - matplotlib - proj - fiona <= 1.18.20 # Till issue https://github.com/Toblerity/Fiona/issues/1085 is not solved + - country_converter # Keep in conda environment when calling ipython - ipython @@ -50,7 +51,6 @@ dependencies: - geopy - tqdm - pytz - - country_converter - tabula-py - pip: diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 0fb20b06..4add4c85 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -79,6 +79,7 @@ def get_eia_annual_hydro_generation(fn, countries): df.loc["Germany"] = df.filter(like='Germany', axis=0).sum() df.loc["Serbia"] += df.loc["Kosovo"] df = df.loc[~df.index.str.contains('Former')] + df.drop(["Europe", "Germany, West", "Germany, East"], inplace=True) df.index = cc.convert(df.index, to='iso2') df.index.name = 'countries' From 51dffbefa393a2498b8d2fd28aaaf63a051ec1f0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 5 Apr 2022 16:59:51 +0200 Subject: [PATCH 08/61] add gas usage limit constraint --- config.default.yaml | 1 + scripts/prepare_network.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/config.default.yaml b/config.default.yaml index 4cd0eadc..128c7bac 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -39,6 +39,7 @@ enable: electricity: voltages: [220., 300., 380.] + gaslimit: false co2limit: 7.75e+7 # 0.05 * 3.1e9*0.5 co2base: 1.487e+9 agg_p_nom_limits: data/agg_p_nom_minmax.csv diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 206e220b..27aacacf 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -77,6 +77,16 @@ def add_co2limit(n, co2limit, Nyears=1.): constant=co2limit * Nyears) +def add_gaslimit(n, gaslimit, Nyears=1.): + + sel = n.carriers.index.intersection(["OCGT", "CCGT", "CHP"]) + n.carriers.loc[sel, "gas_usage"] = 1. + + n.add("GlobalConstraint", "GasLimit", + carrier_attribute="gas_usage", sense="<=", + constant=gaslimit * Nyears) + + def add_emission_prices(n, emission_prices={'co2': 0.}, exclude_co2=False): if exclude_co2: emission_prices.pop('co2') ep = (pd.Series(emission_prices).rename(lambda x: x+'_emissions') * @@ -237,6 +247,10 @@ if __name__ == "__main__": add_co2limit(n, snakemake.config['electricity']['co2limit'], Nyears) break + gaslimit = snakemake.config["electricity"].get("gaslimit") + if gaslimit: + add_gaslimit(n, gaslimit, Nyears) + for o in opts: oo = o.split("+") suptechs = map(lambda c: c.split("-", 2)[0], n.carriers.index) From c4bb470b933237a114efcaf8a866de7c0ccbc4a9 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 5 Apr 2022 17:05:50 +0200 Subject: [PATCH 09/61] add release note and instructions on global gas limit --- config.default.yaml | 2 +- doc/release_notes.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config.default.yaml b/config.default.yaml index 128c7bac..6983945c 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -39,7 +39,7 @@ enable: electricity: voltages: [220., 300., 380.] - gaslimit: false + gaslimit: false # global gas usage limit of X MWh_th co2limit: 7.75e+7 # 0.05 * 3.1e9*0.5 co2base: 1.487e+9 agg_p_nom_limits: data/agg_p_nom_minmax.csv diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 4ef5e956..78a9997d 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -22,6 +22,8 @@ Energy Security Release (April 2022) * old: ``estimate_renewable_capacities_from_capacity_stats`` * new: ``estimate_renewable_capacities`` +* Add function to add global constraint on use of gas in :mod:`prepare_network`. This can be activated with `electricity: gaslimit:` given in MWh. + * The powerplants that have been shut down before 2021 are filtered out. **Bugs and Compatibility** From 3678e5c523080190468e2ce235663df43e644714 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 7 Apr 2022 14:39:34 +0200 Subject: [PATCH 10/61] Add operational reserve margin constraint analogous to GenX Co-authored-by: FabianHofmann --- config.default.yaml | 5 +++ doc/release_notes.rst | 3 ++ scripts/solve_network.py | 74 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 80 insertions(+), 2 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index 6983945c..8ea82891 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -44,6 +44,11 @@ electricity: co2base: 1.487e+9 agg_p_nom_limits: data/agg_p_nom_minmax.csv + operational_reserve: # like https://genxproject.github.io/GenX/dev/core/#Reserves + epsilon_load: 0.02 # share of total load + epsilon_vres: 0.02 # share of total renewable supply + contingency: 4000 # fixed capacity in MW + extendable_carriers: Generator: [] StorageUnit: [] # battery, H2 diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 78a9997d..97a24291 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -22,6 +22,9 @@ Energy Security Release (April 2022) * old: ``estimate_renewable_capacities_from_capacity_stats`` * new: ``estimate_renewable_capacities`` +* Add operational reserve margin constraint analogous to `GenX implementation `_. + Can be activated with config setting ``electricity: operational_reserve:``. + * Add function to add global constraint on use of gas in :mod:`prepare_network`. This can be activated with `electricity: gaslimit:` given in MWh. * The powerplants that have been shut down before 2021 are filtered out. diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 4704d179..a13b1531 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -84,8 +84,9 @@ import pandas as pd import re import pypsa -from pypsa.linopf import (get_var, define_constraints, linexpr, join_exprs, - network_lopf, ilopf) +from pypsa.linopf import (get_var, define_constraints, define_variables, + linexpr, join_exprs, network_lopf, ilopf) +from pypsa.descriptors import get_switchable_as_dense as get_as_dense from pathlib import Path from vresutils.benchmark import memory_logger @@ -211,6 +212,73 @@ def add_SAFE_constraints(n, config): define_constraints(n, lhs, '>=', rhs, 'Safe', 'mintotalcap') +def add_operational_reserve_margin_constraint(n, config): + + reserve_config = config["electricity"]["operational_reserve"] + EPSILON_LOAD = reserve_config["epsilon_load"] + EPSILON_VRES = reserve_config["epsilon_vres"] + CONTINGENCY = reserve_config["contingency"] + + # Reserve Variables + reserve = get_var(n, 'Generator', 'r') + + # Share of extendable renewable capacities + ext_i = n.generators.query('p_nom_extendable').index + vres_i = n.generators_t.p_max_pu.columns + capacity_factor = n.generators_t.p_max_pu[vres_i.intersection(ext_i)] + renewable_capacity_variables = get_var(n, 'Generator', 'p_nom')[vres_i.intersection(ext_i)] + + # Left-hand-side + lhs = ( + linexpr((1, reserve)).sum(1) + + linexpr((-EPSILON_VRES * capacity_factor, renewable_capacity_variables)).sum(1) + ) + + + # Total demand at t + demand = n.loads_t.p.sum(1) + + # VRES potential of non extendable generators + capacity_factor = n.generators_t.p_max_pu[vres_i.difference(ext_i)] + renewable_capacity = n.generators.p_nom[vres_i.difference(ext_i)] + potential = (capacity_factor * renewable_capacity).sum(1) + + # Right-hand-side + rhs = EPSILON_LOAD * demand + EPSILON_VRES * potential + CONTINGENCY + + define_constraints(n, lhs, '>=', rhs, "Reserve margin") + + +def update_capacity_constraint(n): + gen_i = n.generators.index + ext_i = n.generators.query('p_nom_extendable').index + fix_i = n.generators.query('not p_nom_extendable').index + + dispatch = get_var(n, 'Generator', 'p') + reserve = get_var(n, 'Generator', 'r') + + capacity_variable = get_var(n, 'Generator', 'p_nom') + capacity_fixed = n.generators.p_nom[fix_i] + + p_max_pu = get_as_dense(n, 'Generator', 'p_max_pu') + + lhs = linexpr((1, dispatch), (1, reserve)) + lhs += linexpr((-p_max_pu[ext_i], capacity_variable)).reindex(columns=gen_i, fill_value='') + + rhs = (p_max_pu[fix_i] * capacity_fixed).reindex(columns=gen_i, fill_value=0) + + define_constraints(n, lhs, '<=', rhs, 'Generators', 'updated_capacity_constraint') + + +def add_operational_reserve_margin(n, sns, config): + + define_variables(n, 0, np.inf, 'Generator', 'r', axes=[sns, n.generators.index]) + + add_operational_reserve_margin_constraint(n, config) + + update_capacity_constraint(n) + + def add_battery_constraints(n): nodes = n.buses.index[n.buses.carrier == "battery"] if nodes.empty or ('Link', 'p_nom') not in n.variables.index: @@ -236,6 +304,8 @@ def extra_functionality(n, snapshots): add_SAFE_constraints(n, config) if 'CCL' in opts and n.generators.p_nom_extendable.any(): add_CCL_constraints(n, config) + if config["electricity"].get("operational_reserve"): + add_operational_reserve_margin(n, snapshots, config) for o in opts: if "EQ" in o: add_EQ_constraints(n, o) From 84e146834c412edb6f402255dd1fcdcc77de3cd6 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 7 Apr 2022 15:22:10 +0200 Subject: [PATCH 11/61] Apply suggestion from code review to add switch --- config.default.yaml | 1 + scripts/solve_network.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config.default.yaml b/config.default.yaml index 8ea82891..ecb9f201 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -45,6 +45,7 @@ electricity: agg_p_nom_limits: data/agg_p_nom_minmax.csv operational_reserve: # like https://genxproject.github.io/GenX/dev/core/#Reserves + activate: true epsilon_load: 0.02 # share of total load epsilon_vres: 0.02 # share of total renewable supply contingency: 4000 # fixed capacity in MW diff --git a/scripts/solve_network.py b/scripts/solve_network.py index a13b1531..0398dce0 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -304,7 +304,8 @@ def extra_functionality(n, snapshots): add_SAFE_constraints(n, config) if 'CCL' in opts and n.generators.p_nom_extendable.any(): add_CCL_constraints(n, config) - if config["electricity"].get("operational_reserve"): + reserve = config["electricity"].get("operational_reserve", {}) + if reserve.get("activate"): add_operational_reserve_margin(n, snapshots, config) for o in opts: if "EQ" in o: From 630fb9783f5acb9586f28d9732cc97ecad288524 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 7 Apr 2022 17:20:56 +0200 Subject: [PATCH 12/61] fix to operational reserve margin to work without any extendable gens --- scripts/solve_network.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 0398dce0..5d4bb780 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -221,20 +221,16 @@ def add_operational_reserve_margin_constraint(n, config): # Reserve Variables reserve = get_var(n, 'Generator', 'r') + lhs = linexpr((1, reserve)).sum(1) # Share of extendable renewable capacities ext_i = n.generators.query('p_nom_extendable').index vres_i = n.generators_t.p_max_pu.columns - capacity_factor = n.generators_t.p_max_pu[vres_i.intersection(ext_i)] - renewable_capacity_variables = get_var(n, 'Generator', 'p_nom')[vres_i.intersection(ext_i)] + if not ext_i.empty and not vres_i.empty: + capacity_factor = n.generators_t.p_max_pu[vres_i.intersection(ext_i)] + renewable_capacity_variables = get_var(n, 'Generator', 'p_nom')[vres_i.intersection(ext_i)] + lhs += linexpr((-EPSILON_VRES * capacity_factor, renewable_capacity_variables)).sum(1) - # Left-hand-side - lhs = ( - linexpr((1, reserve)).sum(1) + - linexpr((-EPSILON_VRES * capacity_factor, renewable_capacity_variables)).sum(1) - ) - - # Total demand at t demand = n.loads_t.p.sum(1) @@ -256,14 +252,16 @@ def update_capacity_constraint(n): dispatch = get_var(n, 'Generator', 'p') reserve = get_var(n, 'Generator', 'r') - - capacity_variable = get_var(n, 'Generator', 'p_nom') + capacity_fixed = n.generators.p_nom[fix_i] p_max_pu = get_as_dense(n, 'Generator', 'p_max_pu') lhs = linexpr((1, dispatch), (1, reserve)) - lhs += linexpr((-p_max_pu[ext_i], capacity_variable)).reindex(columns=gen_i, fill_value='') + + if not ext_i.empty: + capacity_variable = get_var(n, 'Generator', 'p_nom') + lhs += linexpr((-p_max_pu[ext_i], capacity_variable)).reindex(columns=gen_i, fill_value='') rhs = (p_max_pu[fix_i] * capacity_fixed).reindex(columns=gen_i, fill_value=0) From 64424ed208812ddac2e3dd6453340acceebf2c48 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 7 Apr 2022 21:13:48 +0200 Subject: [PATCH 13/61] add results dir to simplify multiple configs (like PyPSA-Eur-Sec) --- Snakefile | 70 ++++++++++++++++++++++----------------------- config.default.yaml | 2 +- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/Snakefile b/Snakefile index f3a894fc..37f0ddca 100644 --- a/Snakefile +++ b/Snakefile @@ -15,7 +15,7 @@ configfile: "config.yaml" COSTS="data/costs.csv" ATLITE_NPROCESSES = config['atlite'].get('nprocesses', 4) - +RDIR = config["results_dir"] wildcard_constraints: simpl="[a-zA-Z0-9]*|all", @@ -25,19 +25,19 @@ wildcard_constraints: rule cluster_all_networks: - input: expand("networks/elec_s{simpl}_{clusters}.nc", **config['scenario']) + input: expand(RDIR + "networks/elec_s{simpl}_{clusters}.nc", **config['scenario']) rule extra_components_all_networks: - input: expand("networks/elec_s{simpl}_{clusters}_ec.nc", **config['scenario']) + input: expand(RDIR + "networks/elec_s{simpl}_{clusters}_ec.nc", **config['scenario']) rule prepare_all_networks: - input: expand("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) + input: expand(RDIR + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) rule solve_all_networks: - input: expand("results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) + input: expand(RDIR + "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) if config['enable'].get('prepare_links_p_nom', False): @@ -228,7 +228,7 @@ rule add_electricity: nuts3_shapes='resources/nuts3_shapes.geojson', **{f"profile_{tech}": f"resources/profile_{tech}.nc" for tech in config['renewable']} - output: "networks/elec.nc" + output: RDIR + "/prenetworks/elec.nc" log: "logs/add_electricity.log" benchmark: "benchmarks/add_electricity" threads: 1 @@ -238,16 +238,16 @@ rule add_electricity: rule simplify_network: input: - network='networks/elec.nc', + network=RDIR + '/prenetworks/elec.nc', tech_costs=COSTS, - regions_onshore="resources/regions_onshore.geojson", - regions_offshore="resources/regions_offshore.geojson" + regions_onshore="/resources/regions_onshore.geojson", + regions_offshore="/resources/regions_offshore.geojson" output: - network='networks/elec_s{simpl}.nc', - regions_onshore="resources/regions_onshore_elec_s{simpl}.geojson", - regions_offshore="resources/regions_offshore_elec_s{simpl}.geojson", - busmap='resources/busmap_elec_s{simpl}.csv', - connection_costs='resources/connection_costs_s{simpl}.csv' + network=RDIR + '/prenetworks/elec_s{simpl}.nc', + regions_onshore=RDIR + "/resources/regions_onshore_elec_s{simpl}.geojson", + regions_offshore=RDIR + "/resources/regions_offshore_elec_s{simpl}.geojson", + busmap=RDIR + '/resources/busmap_elec_s{simpl}.csv', + connection_costs=RDIR + '/resources/connection_costs_s{simpl}.csv' log: "logs/simplify_network/elec_s{simpl}.log" benchmark: "benchmarks/simplify_network/elec_s{simpl}" threads: 1 @@ -257,19 +257,19 @@ rule simplify_network: rule cluster_network: input: - network='networks/elec_s{simpl}.nc', - regions_onshore="resources/regions_onshore_elec_s{simpl}.geojson", - regions_offshore="resources/regions_offshore_elec_s{simpl}.geojson", - busmap=ancient('resources/busmap_elec_s{simpl}.csv'), + network=RDIR + '/prenetworks/elec_s{simpl}.nc', + regions_onshore=RDIR + "/resources/regions_onshore_elec_s{simpl}.geojson", + regions_offshore=RDIR + "/resources/regions_offshore_elec_s{simpl}.geojson", + busmap=ancient(RDIR + '/resources/busmap_elec_s{simpl}.csv'), custom_busmap=("data/custom_busmap_elec_s{simpl}_{clusters}.csv" if config["enable"].get("custom_busmap", False) else []), tech_costs=COSTS output: - network='networks/elec_s{simpl}_{clusters}.nc', - regions_onshore="resources/regions_onshore_elec_s{simpl}_{clusters}.geojson", - regions_offshore="resources/regions_offshore_elec_s{simpl}_{clusters}.geojson", - busmap="resources/busmap_elec_s{simpl}_{clusters}.csv", - linemap="resources/linemap_elec_s{simpl}_{clusters}.csv" + network=RDIR + '/prenetworks/elec_s{simpl}_{clusters}.nc', + regions_onshore=RDIR + "/resources/regions_onshore_elec_s{simpl}_{clusters}.geojson", + regions_offshore=RDIR + "/resources/regions_offshore_elec_s{simpl}_{clusters}.geojson", + busmap=RDIR + "/resources/busmap_elec_s{simpl}_{clusters}.csv", + linemap=RDIR + "/resources/linemap_elec_s{simpl}_{clusters}.csv" log: "logs/cluster_network/elec_s{simpl}_{clusters}.log" benchmark: "benchmarks/cluster_network/elec_s{simpl}_{clusters}" threads: 1 @@ -279,9 +279,9 @@ rule cluster_network: rule add_extra_components: input: - network='networks/elec_s{simpl}_{clusters}.nc', + network=RDIR + '/prenetworks/elec_s{simpl}_{clusters}.nc', tech_costs=COSTS, - output: 'networks/elec_s{simpl}_{clusters}_ec.nc' + output: RDIR + '/prenetworks/elec_s{simpl}_{clusters}_ec.nc' log: "logs/add_extra_components/elec_s{simpl}_{clusters}.log" benchmark: "benchmarks/add_extra_components/elec_s{simpl}_{clusters}_ec" threads: 1 @@ -290,8 +290,8 @@ rule add_extra_components: rule prepare_network: - input: 'networks/elec_s{simpl}_{clusters}_ec.nc', tech_costs=COSTS - output: 'networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc' + input: RDIR + '/prenetworks/elec_s{simpl}_{clusters}_ec.nc', tech_costs=COSTS + output: RDIR + '/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc' log: "logs/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.log" benchmark: "benchmarks/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 1 @@ -320,8 +320,8 @@ def memory(w): rule solve_network: - input: "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + input: RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + output: RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" log: solver=normpath("logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"), python="logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", @@ -335,9 +335,9 @@ rule solve_network: rule solve_operations_network: input: - unprepared="networks/elec_s{simpl}_{clusters}_ec.nc", - optimized="results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" + unprepared=RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec.nc", + optimized=RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + output: RDIR + "postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" log: solver=normpath("logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"), python="logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", @@ -351,7 +351,7 @@ rule solve_operations_network: rule plot_network: input: - network="results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + network=RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", tech_costs=COSTS output: only_map="results/plots/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}.{ext}", @@ -369,7 +369,7 @@ def input_make_summary(w): else: ll = w.ll return ([COSTS] + - expand("results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + expand(RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", ll=ll, **{k: config["scenario"][k] if getattr(w, k) == "all" else getattr(w, k) for k in ["simpl", "clusters", "opts"]})) @@ -390,7 +390,7 @@ rule plot_summary: def input_plot_p_nom_max(w): - return [("networks/elec_s{simpl}{maybe_cluster}.nc" + return [(RDIR + "/postnetworks/elec_s{simpl}{maybe_cluster}.nc" .format(maybe_cluster=('' if c == 'full' else ('_' + c)), **w)) for c in w.clusts.split(",")] diff --git a/config.default.yaml b/config.default.yaml index ecb9f201..45903b80 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -9,7 +9,7 @@ logging: level: INFO format: '%(levelname)s:%(name)s:%(message)s' -summary_dir: results +results_dir: results/your-run-name scenario: simpl: [''] From b1143dc39b332020c03241b1981691156b2b447a Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 7 Apr 2022 21:48:00 +0200 Subject: [PATCH 14/61] env: update to powerplantmatching >= v0.5.2 --- envs/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 9772d882..7bcb8163 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -56,4 +56,4 @@ dependencies: - pip: - vresutils>=0.3.1 - tsam>=1.1.0 - - powerplantmatching>=0.5.1 + - powerplantmatching>=0.5.2 From f474f8bce43ac671e9b6a477c45494c9bc0c8dbd Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 7 Apr 2022 22:07:18 +0200 Subject: [PATCH 15/61] finetuning of results dir --- Snakefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Snakefile b/Snakefile index 37f0ddca..87d31cd5 100644 --- a/Snakefile +++ b/Snakefile @@ -25,19 +25,19 @@ wildcard_constraints: rule cluster_all_networks: - input: expand(RDIR + "networks/elec_s{simpl}_{clusters}.nc", **config['scenario']) + input: expand(RDIR + "/prenetworks/elec_s{simpl}_{clusters}.nc", **config['scenario']) rule extra_components_all_networks: - input: expand(RDIR + "networks/elec_s{simpl}_{clusters}_ec.nc", **config['scenario']) + input: expand(RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec.nc", **config['scenario']) rule prepare_all_networks: - input: expand(RDIR + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) + input: expand(RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) rule solve_all_networks: - input: expand(RDIR + "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) + input: expand(RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) if config['enable'].get('prepare_links_p_nom', False): @@ -240,8 +240,8 @@ rule simplify_network: input: network=RDIR + '/prenetworks/elec.nc', tech_costs=COSTS, - regions_onshore="/resources/regions_onshore.geojson", - regions_offshore="/resources/regions_offshore.geojson" + regions_onshore="resources/regions_onshore.geojson", + regions_offshore="resources/regions_offshore.geojson" output: network=RDIR + '/prenetworks/elec_s{simpl}.nc', regions_onshore=RDIR + "/resources/regions_onshore_elec_s{simpl}.geojson", @@ -337,7 +337,7 @@ rule solve_operations_network: input: unprepared=RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec.nc", optimized=RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: RDIR + "postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" + output: RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" log: solver=normpath("logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"), python="logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", From f9ede37a022975b424cfb404c4dd14da59a667e3 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Apr 2022 10:26:16 +0200 Subject: [PATCH 16/61] allow varying marginal cost of carrier in opts wc: CCGT+m2.0 --- scripts/prepare_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 27aacacf..2664d362 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -257,7 +257,7 @@ if __name__ == "__main__": if oo[0].startswith(tuple(suptechs)): carrier = oo[0] # handles only p_nom_max as stores and lines have no potentials - attr_lookup = {"p": "p_nom_max", "c": "capital_cost"} + attr_lookup = {"p": "p_nom_max", "c": "capital_cost", "m": "marginal_cost"} attr = attr_lookup[oo[1][0]] factor = float(oo[1][1:]) if carrier == "AC": # lines do not have carrier From bd75953674a6d02dc43027cc75761a22de2b662e Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Apr 2022 10:26:38 +0200 Subject: [PATCH 17/61] add logging to co2limit2 --- scripts/prepare_network.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 2664d362..52e2eff4 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -243,8 +243,10 @@ if __name__ == "__main__": if len(m) > 0: co2limit = float(m[0]) * snakemake.config['electricity']['co2base'] add_co2limit(n, co2limit, Nyears) + logger.info("Setting CO2 limit according to wildcard value.") else: add_co2limit(n, snakemake.config['electricity']['co2limit'], Nyears) + logger.info("Setting CO2 limit according to config value.") break gaslimit = snakemake.config["electricity"].get("gaslimit") From 02c06017d3f1373c7eb2107b19d5ae2cfba51278 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Apr 2022 10:27:23 +0200 Subject: [PATCH 18/61] prepare: add gas consumption limit through wildcard: CH4L --- scripts/prepare_network.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 52e2eff4..2b4e6113 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -249,9 +249,17 @@ if __name__ == "__main__": logger.info("Setting CO2 limit according to config value.") break - gaslimit = snakemake.config["electricity"].get("gaslimit") - if gaslimit: - add_gaslimit(n, gaslimit, Nyears) + for o in opts: + if "CH4L" in o: + m = re.findall("[0-9]*\.?[0-9]+$", o) + if len(m) > 0: + limit = float(m[0]) * 1e6 + add_gaslimit(n, limit, Nyears) + logger.info("Setting gas usage limit according to wildcard value.") + else: + add_gaslimit(n, snakemake.config["electricity"].get("gaslimit"), Nyears) + logger.info("Setting gas usage limit according to config value.") + break for o in opts: oo = o.split("+") From 2403650be29b60b9a902f7327f23b92697ddc723 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Apr 2022 10:27:55 +0200 Subject: [PATCH 19/61] prepare: allow varying emission prices in opts wc: e.g. Ep80.5 --- scripts/prepare_network.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 2b4e6113..178c6bb3 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -278,8 +278,16 @@ if __name__ == "__main__": sel = c.df.carrier.str.contains(carrier) c.df.loc[sel,attr] *= factor - if 'Ep' in opts: - add_emission_prices(n, snakemake.config['costs']['emission_prices']) + for o in opts: + if 'Ep' in o: + m = re.findall("[0-9]*\.?[0-9]+$", o) + if len(m) > 0: + logger.info("Setting emission prices according to wildcard value.") + add_emission_prices(n, dict(co2=float(m[0]))) + else: + logger.info("Setting emission prices according to config value.") + add_emission_prices(n, snakemake.config['costs']['emission_prices']) + break ll_type, factor = snakemake.wildcards.ll[0], snakemake.wildcards.ll[1:] set_transmission_limit(n, ll_type, factor, costs, Nyears) From e5cb2d34fbf63ee92f18780fe7d4aebcf22a4204 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 8 Apr 2022 11:30:29 +0200 Subject: [PATCH 20/61] env: update to powerplantmatching >= 0.5.3 --- envs/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 7bcb8163..9c50d8dd 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -56,4 +56,4 @@ dependencies: - pip: - vresutils>=0.3.1 - tsam>=1.1.0 - - powerplantmatching>=0.5.2 + - powerplantmatching>=0.5.3 From e37a3f57ab7fb0e187bacbcd6085b4988df06dc8 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 8 Apr 2022 12:23:24 +0200 Subject: [PATCH 21/61] adjust biomass capacities --- config.default.yaml | 2 +- scripts/build_powerplants.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/config.default.yaml b/config.default.yaml index ecb9f201..e62c6c3c 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -61,7 +61,7 @@ electricity: H2: 168 # use pandas query strings here, e.g. Country not in ['Germany'] - powerplants_filter: (DateOut >= 2021 or DateOut != DateOut) + powerplants_filter: (DateOut >= 2022 or DateOut != DateOut) # use pandas query strings here, e.g. Country in ['Germany'] custom_powerplants: false conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index 9a7c9e23..00b1a9a3 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -115,6 +115,14 @@ if __name__ == "__main__": .replace({'Technology': {'Steam Turbine': 'OCGT', "Combustion Engine": "OCGT"}}) .assign(Fueltype=replace_natural_gas_by_technology)) + # Correct bioenergy for countries where possible + opsd = pm.data.OPSD_VRE().powerplant.convert_country_to_alpha2() + opsd = opsd.query('Country in @countries and Fueltype == "Bioenergy"') + opsd['Fueltype'] = 'biomass' + available_countries = opsd.Country.unique() + ppl = ppl.query('not (Country in @available_countries and Fueltype == "Bioenergy")') + ppl = pd.concat([ppl, opsd]) + ppl_query = snakemake.config['electricity']['powerplants_filter'] if isinstance(ppl_query, str): ppl.query(ppl_query, inplace=True) From 40c882f0e968f45fc3b80beff1cfa7c24f70e9d4 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Apr 2022 13:19:25 +0200 Subject: [PATCH 22/61] solve: allow to parse load shedding cost in config --- scripts/solve_network.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 5d4bb780..a5ebfe6e 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -100,17 +100,19 @@ def prepare_network(n, solve_opts): for df in (n.generators_t.p_max_pu, n.storage_units_t.inflow): df.where(df>solve_opts['clip_p_max_pu'], other=0., inplace=True) - if solve_opts.get('load_shedding'): + load_shedding = solve_opts.get('load_shedding') + if load_shedding: n.add("Carrier", "Load") buses_i = n.buses.query("carrier == 'AC'").index + if not np.isscalar(load_shedding): load_shedding = 1e2 + # intersect between macroeconomic and surveybased + # willingness to pay + # http://journal.frontiersin.org/article/10.3389/fenrg.2015.00055/full) n.madd("Generator", buses_i, " load", bus=buses_i, carrier='load', sign=1e-3, # Adjust sign to measure p and p_nom in kW instead of MW - marginal_cost=1e2, # Eur/kWh - # intersect between macroeconomic and surveybased - # willingness to pay - # http://journal.frontiersin.org/article/10.3389/fenrg.2015.00055/full + marginal_cost=load_shedding, p_nom=1e9 # kW ) From 881c822437a972cd49bfd30503e677e0142d9e5e Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Apr 2022 13:20:43 +0200 Subject: [PATCH 23/61] Snakefile: add powerplants to results-dir --- Snakefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Snakefile b/Snakefile index 87d31cd5..b385a0db 100644 --- a/Snakefile +++ b/Snakefile @@ -221,7 +221,7 @@ rule add_electricity: base_network='networks/base.nc', tech_costs=COSTS, regions="resources/regions_onshore.geojson", - powerplants='resources/powerplants.csv', + powerplants=RDIR + '/resources/powerplants.csv', hydro_capacities='data/bundle/hydro_capacities.csv', geth_hydro_capacities='data/geth2015_hydro_capacities.csv', load='resources/load.csv', @@ -268,7 +268,7 @@ rule cluster_network: network=RDIR + '/prenetworks/elec_s{simpl}_{clusters}.nc', regions_onshore=RDIR + "/resources/regions_onshore_elec_s{simpl}_{clusters}.geojson", regions_offshore=RDIR + "/resources/regions_offshore_elec_s{simpl}_{clusters}.geojson", - busmap=RDIR + "/resources/busmap_elec_s{simpl}_{clusters}.csv", + busmap=RDIR + "/resources/busmap_elec_s{simpl}_{clusters}.csv", linemap=RDIR + "/resources/linemap_elec_s{simpl}_{clusters}.csv" log: "logs/cluster_network/elec_s{simpl}_{clusters}.log" benchmark: "benchmarks/cluster_network/elec_s{simpl}_{clusters}" From f0d0edcf542c5db84b6a02ae838fb489940b5563 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Apr 2022 13:20:43 +0200 Subject: [PATCH 24/61] Snakefile: add powerplants to results-dir --- Snakefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Snakefile b/Snakefile index 87d31cd5..df3f7cac 100644 --- a/Snakefile +++ b/Snakefile @@ -84,7 +84,7 @@ rule build_powerplants: input: base_network="networks/base.nc", custom_powerplants="data/custom_powerplants.csv" - output: "resources/powerplants.csv" + output: RDIR + "/resources/powerplants.csv" log: "logs/build_powerplants.log" threads: 1 resources: mem_mb=500 @@ -221,7 +221,7 @@ rule add_electricity: base_network='networks/base.nc', tech_costs=COSTS, regions="resources/regions_onshore.geojson", - powerplants='resources/powerplants.csv', + powerplants=RDIR + '/resources/powerplants.csv', hydro_capacities='data/bundle/hydro_capacities.csv', geth_hydro_capacities='data/geth2015_hydro_capacities.csv', load='resources/load.csv', @@ -268,7 +268,7 @@ rule cluster_network: network=RDIR + '/prenetworks/elec_s{simpl}_{clusters}.nc', regions_onshore=RDIR + "/resources/regions_onshore_elec_s{simpl}_{clusters}.geojson", regions_offshore=RDIR + "/resources/regions_offshore_elec_s{simpl}_{clusters}.geojson", - busmap=RDIR + "/resources/busmap_elec_s{simpl}_{clusters}.csv", + busmap=RDIR + "/resources/busmap_elec_s{simpl}_{clusters}.csv", linemap=RDIR + "/resources/linemap_elec_s{simpl}_{clusters}.csv" log: "logs/cluster_network/elec_s{simpl}_{clusters}.log" benchmark: "benchmarks/cluster_network/elec_s{simpl}_{clusters}" From d53487f822e717e5bbe3a1e7d93527e84df70176 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 8 Apr 2022 14:23:33 +0200 Subject: [PATCH 25/61] build_powerplants: fix duplicated names per bus --- scripts/build_powerplants.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index 00b1a9a3..cc01d373 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -118,7 +118,7 @@ if __name__ == "__main__": # Correct bioenergy for countries where possible opsd = pm.data.OPSD_VRE().powerplant.convert_country_to_alpha2() opsd = opsd.query('Country in @countries and Fueltype == "Bioenergy"') - opsd['Fueltype'] = 'biomass' + opsd['Name'] = "Biomass" available_countries = opsd.Country.unique() ppl = ppl.query('not (Country in @available_countries and Fueltype == "Bioenergy")') ppl = pd.concat([ppl, opsd]) @@ -148,4 +148,8 @@ if __name__ == "__main__": if bus_null_b.any(): logging.warning(f"Couldn't find close bus for {bus_null_b.sum()} powerplants") + # TODO: This has to fixed in PPM, some powerplants are still duplicated + cumcount = ppl.groupby(['bus', 'Fueltype']).cumcount() + 1 + ppl.Name = ppl.Name.where(cumcount == 1, ppl.Name + " " + cumcount.astype(str)) + ppl.to_csv(snakemake.output[0]) From 65790cd065f32934739a51248bcaf74ba0bdb760 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 8 Apr 2022 14:34:52 +0200 Subject: [PATCH 26/61] build_powerplants: remove non-assigned ppls --- scripts/build_powerplants.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index cc01d373..74f53d80 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -146,7 +146,9 @@ if __name__ == "__main__": bus_null_b = ppl["bus"].isnull() if bus_null_b.any(): - logging.warning(f"Couldn't find close bus for {bus_null_b.sum()} powerplants") + logging.warning(f"Couldn't find close bus for {bus_null_b.sum()} powerplants. " + "Removing them from the powerplants list.") + ppl = ppl[~bus_null_b] # TODO: This has to fixed in PPM, some powerplants are still duplicated cumcount = ppl.groupby(['bus', 'Fueltype']).cumcount() + 1 From 40425a7767e614163aae9c393e2ae6ae84129b59 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 8 Apr 2022 15:25:05 +0200 Subject: [PATCH 27/61] build powerplants: use map_country bus function for bus attachement --- scripts/build_powerplants.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index 74f53d80..c1ee4127 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -79,6 +79,7 @@ import powerplantmatching as pm import pandas as pd import numpy as np +from powerplantmatching.export import map_country_bus from scipy.spatial import cKDTree as KDTree logger = logging.getLogger(__name__) @@ -87,8 +88,7 @@ logger = logging.getLogger(__name__) def add_custom_powerplants(ppl, custom_powerplants, custom_ppl_query=False): if not custom_ppl_query: return ppl - add_ppls = pd.read_csv(custom_powerplants, index_col=0, - dtype={'bus': 'str'}) + add_ppls = pd.read_csv(custom_powerplants, index_col=0, dtype={'bus': 'str'}) if isinstance(custom_ppl_query, str): add_ppls.query(custom_ppl_query, inplace=True) return pd.concat([ppl, add_ppls], sort=False, ignore_index=True, verify_integrity=True) @@ -131,18 +131,12 @@ if __name__ == "__main__": custom_ppl_query = snakemake.config['electricity']['custom_powerplants'] ppl = add_custom_powerplants(ppl, snakemake.input.custom_powerplants, custom_ppl_query) - cntries_without_ppl = [c for c in countries if c not in ppl.Country.unique()] + countries_wo_ppl = [c for c in countries if c not in ppl.Country.unique()] + if countries_wo_ppl: + logging.warning(f"No powerplants known in: {', '.join(countries_wo_ppl)}") - for c in countries: - substation_i = n.buses.query('substation_lv and country == @c').index - kdtree = KDTree(n.buses.loc[substation_i, ['x','y']].values) - ppl_i = ppl.query('Country == @c').index - - tree_i = kdtree.query(ppl.loc[ppl_i, ['lon','lat']].values)[1] - ppl.loc[ppl_i, 'bus'] = substation_i.append(pd.Index([np.nan]))[tree_i] - - if cntries_without_ppl: - logging.warning(f"No powerplants known in: {', '.join(cntries_without_ppl)}") + substations = n.buses.query('substation_lv') + ppl = map_country_bus(ppl, substations) bus_null_b = ppl["bus"].isnull() if bus_null_b.any(): @@ -154,4 +148,4 @@ if __name__ == "__main__": cumcount = ppl.groupby(['bus', 'Fueltype']).cumcount() + 1 ppl.Name = ppl.Name.where(cumcount == 1, ppl.Name + " " + cumcount.astype(str)) - ppl.to_csv(snakemake.output[0]) + ppl.reset_index(drop=True).to_csv(snakemake.output[0]) From c412a61013feae39d3c3451a2fbd938f6c65368e Mon Sep 17 00:00:00 2001 From: euronion <42553970+euronion@users.noreply.github.com> Date: Fri, 8 Apr 2022 15:41:23 +0200 Subject: [PATCH 28/61] Add country-specific EAF restriction for NPPs. (#361) * Add country-specific EAF restriction for NPPs. Based on historic figures from IAEA. * Update release_notes.rst --- config.default.yaml | 25 ++++++++++++++++++++++++- doc/release_notes.rst | 5 ++++- scripts/add_electricity.py | 22 +++++++++++++++++++--- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index e62c6c3c..4f1708b4 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -64,7 +64,30 @@ electricity: powerplants_filter: (DateOut >= 2022 or DateOut != DateOut) # use pandas query strings here, e.g. Country in ['Germany'] custom_powerplants: false - conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] + conventional_carriers: + technologies: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] + # Limit energy availability from these sources -> p_max_pu + # syntax: + # : or : + energy_availability_factors: + # From IAEA + # https://pris.iaea.org/PRIS/WorldStatistics/ThreeYrsEnergyAvailabilityFactor.aspx (2022-04-08) + nuclear: + BE: 0.65 + BG: 0.89 + CZ: 0.82 + FI: 0.92 + FR: 0.70 + DE: 0.88 + HU: 0.90 + NL: 0.86 + RO: 0.92 + SK: 0.89 + SI: 0.94 + ES: 0.89 + SE: 0.82 + CH: 0.86 + GB: 0.67 renewable_capacities_from_OPSD: [] # onwind, offwind, solar estimate_renewable_capacities: diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 97a24291..f07176ad 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -25,7 +25,10 @@ Energy Security Release (April 2022) * Add operational reserve margin constraint analogous to `GenX implementation `_. Can be activated with config setting ``electricity: operational_reserve:``. -* Add function to add global constraint on use of gas in :mod:`prepare_network`. This can be activated with `electricity: gaslimit:` given in MWh. +* Add function to add global constraint on use of gas in :mod:`prepare_network`. This can be activated with `electricity: gaslimit:` given in MWh. + +* Add configuration option to implement Energy Availability Factors (EAFs) for conventional generation technologies. +* Implement country-specific EAFs for nuclear power plants based on IAEA 2018-2020 reported country averages. * The powerplants that have been shut down before 2021 are filtered out. diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 961b87dc..ab3aa321 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -290,8 +290,9 @@ def attach_wind_and_solar(n, costs, input_profiles, technologies, line_length_fa p_max_pu=ds['profile'].transpose('time', 'bus').to_pandas()) -def attach_conventional_generators(n, costs, ppl, carriers): +def attach_conventional_generators(n, costs, ppl, conventional_carriers): + carriers = conventional_carriers["technologies"] _add_missing_carriers_from_costs(n, costs, carriers) ppl = (ppl.query('carrier in @carriers').join(costs, on='carrier') @@ -309,6 +310,22 @@ def attach_conventional_generators(n, costs, ppl, carriers): capital_cost=0) logger.warning(f'Capital costs for conventional generators put to 0 EUR/MW.') + + for k,v in conventional_carriers["energy_availability_factors"].items(): + + # Generators with technology affected + idx = n.generators.query("carrier == @k").index + + if isinstance(v, float): + # Single value affecting all generators of technology k indiscriminantely of country + n.generators.loc[idx, "p_max_pu"] = v + elif isinstance(v, dict): + v = pd.Series(v) + + # Values affecting generators of technology k country-specific + # First map generator buses to countries; then map countries to p_max_pu + n.generators["p_max_pu"] = n.generators.loc[idx]["bus"].replace(n.buses["country"]).replace(v) + def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **config): @@ -556,8 +573,7 @@ if __name__ == "__main__": update_transmission_costs(n, costs, snakemake.config['lines']['length_factor']) - carriers = snakemake.config['electricity']['conventional_carriers'] - attach_conventional_generators(n, costs, ppl, carriers) + attach_conventional_generators(n, costs, ppl, snakemake.config["electricity"]["conventional_carriers"]) carriers = snakemake.config['renewable'] attach_wind_and_solar(n, costs, snakemake.input, carriers, snakemake.config['lines']['length_factor']) From 4712bfc893f8e32816d409a06f293a6e2f129b66 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Apr 2022 19:55:40 +0200 Subject: [PATCH 29/61] fix nuclear EAF processing code --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index ab3aa321..6c8b6de6 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -324,7 +324,7 @@ def attach_conventional_generators(n, costs, ppl, conventional_carriers): # Values affecting generators of technology k country-specific # First map generator buses to countries; then map countries to p_max_pu - n.generators["p_max_pu"] = n.generators.loc[idx]["bus"].replace(n.buses["country"]).replace(v) + n.generators.loc[idx, "p_max_pu"] = n.generators.loc[idx]["bus"].replace(n.buses["country"]).replace(v) From 732d114416d99b9402fea656611e6a44c201ea5b Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Apr 2022 19:59:27 +0200 Subject: [PATCH 30/61] add results_dir to all configs --- config.tutorial.yaml | 2 +- test/config.test1.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 225d8f78..3238329c 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -9,7 +9,7 @@ logging: level: INFO format: '%(levelname)s:%(name)s:%(message)s' -summary_dir: results +results_dir: results/your-run-name scenario: simpl: [''] diff --git a/test/config.test1.yaml b/test/config.test1.yaml index a9ce1e50..600a3aac 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -8,7 +8,7 @@ logging: level: INFO format: '%(levelname)s:%(name)s:%(message)s' -summary_dir: results +results_dir: results/your-run-name scenario: simpl: [''] From 8b855f04787b4959cfa7f22cdb1a2c3675a71155 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 7 Jun 2022 15:17:49 +0200 Subject: [PATCH 31/61] add_elecitricity: scale only missing renewable capacities from OPSD --- scripts/add_electricity.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 6c8b6de6..b2a6f7f4 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -478,18 +478,16 @@ def attach_extendable_generators(n, costs, ppl, carriers): def attach_OPSD_renewables(n, techs): - available = ['DE', 'FR', 'PL', 'CH', 'DK', 'CZ', 'SE', 'GB'] tech_map = {'Onshore': 'onwind', 'Offshore': 'offwind', 'Solar': 'solar'} - countries = set(available) & set(n.buses.country) tech_map = {k: v for k, v in tech_map.items() if v in techs} if not tech_map: return - logger.info(f'Using OPSD renewable capacities in {", ".join(countries)} ' - f'for technologies {", ".join(tech_map.values())}.') + tech_string = ", ".join(tech_map.values()) + logger.info(f'Using OPSD renewable capacities for technologies {tech_string}.') - df = pd.concat([pm.data.OPSD_VRE_country(c) for c in countries]) + df = pm.data.OPSD_VRE().powerplant.convert_country_to_alpha2() technology_b = ~df.Technology.isin(['Onshore', 'Offshore']) df['Fueltype'] = df.Fueltype.where(technology_b, df.Technology) df = df.query('Fueltype in @tech_map').powerplant.convert_country_to_alpha2() @@ -528,21 +526,27 @@ def estimate_renewable_capacities(n, config): logger.info(f"Heuristics applied to distribute renewable capacities [MW] " f"{capacities.groupby('Country').sum()}") + for ppm_technology, techs in tech_map.items(): - tech_capacities = capacities.loc[ppm_technology].reindex(countries, fill_value=0.) tech_i = n.generators.query('carrier in @techs').index - n.generators.loc[tech_i, 'p_nom'] = ( - (n.generators_t.p_max_pu[tech_i].mean() * - n.generators.loc[tech_i, 'p_nom_max']) # maximal yearly generation - .groupby(n.generators.bus.map(n.buses.country)) - .transform(lambda s: normed(s) * tech_capacities.at[s.name]) - .where(lambda s: s>0.1, 0.)) # only capacities above 100kW + stats = capacities.loc[ppm_technology].reindex(countries, fill_value=0.) + country = n.generators[tech_i].bus.map(n.buses.country) + existent = n.generators.p_nom[tech_i].groupby(country).sum() + missing = stats - existent + dist = n.generators_t.p_max_pu.mean() * n.generators.p_nom_max + + n.generators.loc[tech_i, 'p_nom'] += ( + dist[tech_i] + .groupby(country) + .transform(lambda s: normed(s) * missing[s.name]) + .where(lambda s: s>0.1, 0.) # only capacities above 100kW + ) n.generators.loc[tech_i, 'p_nom_min'] = n.generators.loc[tech_i, 'p_nom'] if expansion_limit: assert np.isscalar(expansion_limit) logger.info(f"Reducing capacity expansion limit to {expansion_limit*100:.2f}% of installed capacity.") - n.generators.loc[tech_i, 'p_nom_max'] = float(expansion_limit) * n.generators.loc[tech_i, 'p_nom_min'] + n.generators.loc[tech_i, 'p_nom_max'] = expansion_limit * n.generators.loc[tech_i, 'p_nom_min'] def add_nice_carrier_names(n, config): @@ -586,9 +590,11 @@ if __name__ == "__main__": carriers = snakemake.config['electricity']['extendable_carriers']['Generator'] attach_extendable_generators(n, costs, ppl, carriers) - estimate_renewable_capacities(n, snakemake.config) techs = snakemake.config['electricity'].get('renewable_capacities_from_OPSD', []) attach_OPSD_renewables(n, techs) + + estimate_renewable_capacities(n, snakemake.config) + update_p_nom_max(n) add_nice_carrier_names(n, snakemake.config) From eb59e68f353f4bf141cf315e834d8070691231c2 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 9 Jun 2022 20:24:28 +0200 Subject: [PATCH 32/61] Snakefile: remove RDIR selection --- Snakefile | 69 +++++++++++++++++++++++++++---------------------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/Snakefile b/Snakefile index df3f7cac..7df3a6ac 100644 --- a/Snakefile +++ b/Snakefile @@ -15,7 +15,6 @@ configfile: "config.yaml" COSTS="data/costs.csv" ATLITE_NPROCESSES = config['atlite'].get('nprocesses', 4) -RDIR = config["results_dir"] wildcard_constraints: simpl="[a-zA-Z0-9]*|all", @@ -25,19 +24,19 @@ wildcard_constraints: rule cluster_all_networks: - input: expand(RDIR + "/prenetworks/elec_s{simpl}_{clusters}.nc", **config['scenario']) + input: expand("networks/elec_s{simpl}_{clusters}.nc", **config['scenario']) rule extra_components_all_networks: - input: expand(RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec.nc", **config['scenario']) + input: expand("networks/elec_s{simpl}_{clusters}_ec.nc", **config['scenario']) rule prepare_all_networks: - input: expand(RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) + input: expand("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) rule solve_all_networks: - input: expand(RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) + input: expand("results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) if config['enable'].get('prepare_links_p_nom', False): @@ -84,7 +83,7 @@ rule build_powerplants: input: base_network="networks/base.nc", custom_powerplants="data/custom_powerplants.csv" - output: RDIR + "/resources/powerplants.csv" + output: "resources/powerplants.csv" log: "logs/build_powerplants.log" threads: 1 resources: mem_mb=500 @@ -221,14 +220,14 @@ rule add_electricity: base_network='networks/base.nc', tech_costs=COSTS, regions="resources/regions_onshore.geojson", - powerplants=RDIR + '/resources/powerplants.csv', + powerplants='resources/powerplants.csv', hydro_capacities='data/bundle/hydro_capacities.csv', geth_hydro_capacities='data/geth2015_hydro_capacities.csv', load='resources/load.csv', nuts3_shapes='resources/nuts3_shapes.geojson', **{f"profile_{tech}": f"resources/profile_{tech}.nc" for tech in config['renewable']} - output: RDIR + "/prenetworks/elec.nc" + output: "networks/elec.nc" log: "logs/add_electricity.log" benchmark: "benchmarks/add_electricity" threads: 1 @@ -238,16 +237,16 @@ rule add_electricity: rule simplify_network: input: - network=RDIR + '/prenetworks/elec.nc', + network='networks/elec.nc', tech_costs=COSTS, regions_onshore="resources/regions_onshore.geojson", regions_offshore="resources/regions_offshore.geojson" output: - network=RDIR + '/prenetworks/elec_s{simpl}.nc', - regions_onshore=RDIR + "/resources/regions_onshore_elec_s{simpl}.geojson", - regions_offshore=RDIR + "/resources/regions_offshore_elec_s{simpl}.geojson", - busmap=RDIR + '/resources/busmap_elec_s{simpl}.csv', - connection_costs=RDIR + '/resources/connection_costs_s{simpl}.csv' + network='networks/elec_s{simpl}.nc', + regions_onshore="resources/regions_onshore_elec_s{simpl}.geojson", + regions_offshore="resources/regions_offshore_elec_s{simpl}.geojson", + busmap='resources/busmap_elec_s{simpl}.csv', + connection_costs='resources/connection_costs_s{simpl}.csv' log: "logs/simplify_network/elec_s{simpl}.log" benchmark: "benchmarks/simplify_network/elec_s{simpl}" threads: 1 @@ -257,19 +256,19 @@ rule simplify_network: rule cluster_network: input: - network=RDIR + '/prenetworks/elec_s{simpl}.nc', - regions_onshore=RDIR + "/resources/regions_onshore_elec_s{simpl}.geojson", - regions_offshore=RDIR + "/resources/regions_offshore_elec_s{simpl}.geojson", - busmap=ancient(RDIR + '/resources/busmap_elec_s{simpl}.csv'), + network='networks/elec_s{simpl}.nc', + regions_onshore="resources/regions_onshore_elec_s{simpl}.geojson", + regions_offshore="resources/regions_offshore_elec_s{simpl}.geojson", + busmap=ancient('resources/busmap_elec_s{simpl}.csv'), custom_busmap=("data/custom_busmap_elec_s{simpl}_{clusters}.csv" if config["enable"].get("custom_busmap", False) else []), tech_costs=COSTS output: - network=RDIR + '/prenetworks/elec_s{simpl}_{clusters}.nc', - regions_onshore=RDIR + "/resources/regions_onshore_elec_s{simpl}_{clusters}.geojson", - regions_offshore=RDIR + "/resources/regions_offshore_elec_s{simpl}_{clusters}.geojson", - busmap=RDIR + "/resources/busmap_elec_s{simpl}_{clusters}.csv", - linemap=RDIR + "/resources/linemap_elec_s{simpl}_{clusters}.csv" + network='networks/elec_s{simpl}_{clusters}.nc', + regions_onshore="resources/regions_onshore_elec_s{simpl}_{clusters}.geojson", + regions_offshore="resources/regions_offshore_elec_s{simpl}_{clusters}.geojson", + busmap="resources/busmap_elec_s{simpl}_{clusters}.csv", + linemap="resources/linemap_elec_s{simpl}_{clusters}.csv" log: "logs/cluster_network/elec_s{simpl}_{clusters}.log" benchmark: "benchmarks/cluster_network/elec_s{simpl}_{clusters}" threads: 1 @@ -279,9 +278,9 @@ rule cluster_network: rule add_extra_components: input: - network=RDIR + '/prenetworks/elec_s{simpl}_{clusters}.nc', + network='networks/elec_s{simpl}_{clusters}.nc', tech_costs=COSTS, - output: RDIR + '/prenetworks/elec_s{simpl}_{clusters}_ec.nc' + output: 'networks/elec_s{simpl}_{clusters}_ec.nc' log: "logs/add_extra_components/elec_s{simpl}_{clusters}.log" benchmark: "benchmarks/add_extra_components/elec_s{simpl}_{clusters}_ec" threads: 1 @@ -290,8 +289,8 @@ rule add_extra_components: rule prepare_network: - input: RDIR + '/prenetworks/elec_s{simpl}_{clusters}_ec.nc', tech_costs=COSTS - output: RDIR + '/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc' + input: 'networks/elec_s{simpl}_{clusters}_ec.nc', tech_costs=COSTS + output: 'networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc' log: "logs/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.log" benchmark: "benchmarks/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 1 @@ -320,8 +319,8 @@ def memory(w): rule solve_network: - input: RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + input: "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" log: solver=normpath("logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"), python="logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", @@ -335,9 +334,9 @@ rule solve_network: rule solve_operations_network: input: - unprepared=RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec.nc", - optimized=RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" + unprepared="networks/elec_s{simpl}_{clusters}_ec.nc", + optimized="results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" log: solver=normpath("logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"), python="logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", @@ -351,7 +350,7 @@ rule solve_operations_network: rule plot_network: input: - network=RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + network="results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", tech_costs=COSTS output: only_map="results/plots/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}.{ext}", @@ -369,7 +368,7 @@ def input_make_summary(w): else: ll = w.ll return ([COSTS] + - expand(RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + expand("results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", ll=ll, **{k: config["scenario"][k] if getattr(w, k) == "all" else getattr(w, k) for k in ["simpl", "clusters", "opts"]})) @@ -390,7 +389,7 @@ rule plot_summary: def input_plot_p_nom_max(w): - return [(RDIR + "/postnetworks/elec_s{simpl}{maybe_cluster}.nc" + return [("results/networks/elec_s{simpl}{maybe_cluster}.nc" .format(maybe_cluster=('' if c == 'full' else ('_' + c)), **w)) for c in w.clusts.split(",")] From 0ec3a8638b1c37e100e900de746f537248c3551e Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 9 Jun 2022 20:31:50 +0200 Subject: [PATCH 33/61] add_electricity & config: - refactor attachment of conventional carriers - refactor scaling of renewable carriers --- config.default.yaml | 70 ++++++++++---------- scripts/add_electricity.py | 131 +++++++++++++++++++++---------------- 2 files changed, 107 insertions(+), 94 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index edd88567..fc09dfd6 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -9,8 +9,6 @@ logging: level: INFO format: '%(levelname)s:%(name)s:%(message)s' -results_dir: results/your-run-name - scenario: simpl: [''] ll: ['copt'] @@ -50,57 +48,36 @@ electricity: epsilon_vres: 0.02 # share of total renewable supply contingency: 4000 # fixed capacity in MW - extendable_carriers: - Generator: [] - StorageUnit: [] # battery, H2 - Store: [battery, H2] - Link: [] - max_hours: battery: 6 H2: 168 + extendable_carriers: + Generator: [solar, onwind, offwind-ac, offwind-dc, OCGT] + StorageUnit: [] # battery, H2 + Store: [battery, H2] + Link: [AC, DC] + # use pandas query strings here, e.g. Country not in ['Germany'] powerplants_filter: (DateOut >= 2022 or DateOut != DateOut) # use pandas query strings here, e.g. Country in ['Germany'] custom_powerplants: false - conventional_carriers: - technologies: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] - # Limit energy availability from these sources -> p_max_pu - # syntax: - # : or : - energy_availability_factors: - # From IAEA - # https://pris.iaea.org/PRIS/WorldStatistics/ThreeYrsEnergyAvailabilityFactor.aspx (2022-04-08) - nuclear: - BE: 0.65 - BG: 0.89 - CZ: 0.82 - FI: 0.92 - FR: 0.70 - DE: 0.88 - HU: 0.90 - NL: 0.86 - RO: 0.92 - SK: 0.89 - SI: 0.94 - ES: 0.89 - SE: 0.82 - CH: 0.86 - GB: 0.67 - renewable_capacities_from_OPSD: [] # onwind, offwind, solar + + conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] + renewable_carriers: [solar, onwind, offwind-ac, offwind-dc, hydro] estimate_renewable_capacities: + enable: true + # Add capacities from OPSD data + from_opsd: true # Renewable capacities are based on existing capacities reported by IRENA - - # Reference year, any of 2000 to 2020 year: 2020 # Artificially limit maximum capacities to factor * (IRENA capacities), # i.e. 110% of 's capacities => expansion_limit: 1.1 # false: Use estimated renewable potentials determine by the workflow expansion_limit: false technology_mapping: - # Wind is the Fueltype in ppm.data.Capacity_stats, onwind, offwind-{ac,dc} the carrier in PyPSA-Eur + # Wind is the Fueltype in powerplantmatching, onwind, offwind-{ac,dc} the carrier in PyPSA-Eur Offshore: [offwind-ac, offwind-dc] Onshore: [onwind] PV: [solar] @@ -210,6 +187,27 @@ renewable: hydro_max_hours: "energy_capacity_totals_by_country" # one of energy_capacity_totals_by_country, estimate_by_large_installations or a float clip_min_inflow: 1.0 +conventional: + nuclear: + energy_availability_factors: + # From IAEA + # https://pris.iaea.org/PRIS/WorldStatistics/ThreeYrsEnergyAvailabilityFactor.aspx (2022-04-08) + BE: 0.65 + BG: 0.89 + CZ: 0.82 + FI: 0.92 + FR: 0.70 + DE: 0.88 + HU: 0.90 + NL: 0.86 + RO: 0.92 + SK: 0.89 + SI: 0.94 + ES: 0.89 + SE: 0.82 + CH: 0.86 + GB: 0.67 + lines: types: 220.: "Al/St 240/40 2-bundle 220.0" diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index b2a6f7f4..992c0b3d 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -24,8 +24,8 @@ Relevant Settings conventional_carriers: co2limit: extendable_carriers: - include_renewable_capacities_from_OPSD: - estimate_renewable_capacities_from_capacity_stats: + estimate_renewable_capacities: + load: scaling_factor: @@ -185,7 +185,7 @@ def load_powerplants(ppl_fn): 'ccgt, thermal': 'CCGT', 'hard coal': 'coal'} return (pd.read_csv(ppl_fn, index_col=0, dtype={'bus': 'str'}) .powerplant.to_pypsa_names() - .rename(columns=str.lower).drop(columns=['efficiency']) + .rename(columns=str.lower) .replace({'carrier': carrier_dict})) @@ -251,13 +251,14 @@ def update_transmission_costs(n, costs, length_factor=1.0): n.links.loc[dc_b, 'capital_cost'] = costs -def attach_wind_and_solar(n, costs, input_profiles, technologies, line_length_factor=1): +def attach_wind_and_solar(n, costs, input_profiles, technologies, extendable_carriers, line_length_factor=1): # TODO: rename tech -> carrier, technologies -> carriers - - for tech in technologies: - if tech == 'hydro': continue + _add_missing_carriers_from_costs(n, costs, technologies) + + for tech in technologies: + if tech == 'hydro': + continue - n.add("Carrier", name=tech) with xr.open_dataset(getattr(input_profiles, 'profile_' + tech)) as ds: if ds.indexes['bus'].empty: continue @@ -281,7 +282,7 @@ def attach_wind_and_solar(n, costs, input_profiles, technologies, line_length_fa n.madd("Generator", ds.indexes['bus'], ' ' + tech, bus=ds.indexes['bus'], carrier=tech, - p_nom_extendable=True, + p_nom_extendable=tech in extendable_carriers['Generator'], p_nom_max=ds['p_nom_max'].to_pandas(), weight=ds['weight'].to_pandas(), marginal_cost=costs.at[suptech, 'marginal_cost'], @@ -290,41 +291,45 @@ def attach_wind_and_solar(n, costs, input_profiles, technologies, line_length_fa p_max_pu=ds['profile'].transpose('time', 'bus').to_pandas()) -def attach_conventional_generators(n, costs, ppl, conventional_carriers): +def attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers, **config): - carriers = conventional_carriers["technologies"] + carriers = set(conventional_carriers) | set(extendable_carriers['Generator']) _add_missing_carriers_from_costs(n, costs, carriers) - ppl = (ppl.query('carrier in @carriers').join(costs, on='carrier') + ppl = (ppl.query('carrier in @carriers').join(costs, on='carrier', rsuffix='_r') .rename(index=lambda s: 'C' + str(s))) + ppl.efficiency.update(ppl.efficiency_r.dropna()) - logger.info('Adding {} generators with capacities [MW] \n{}' - .format(len(ppl), ppl.groupby('carrier').p_nom.sum())) + logger.info('Adding {} generators with capacities [GW] \n{}' + .format(len(ppl), ppl.groupby('carrier').p_nom.sum().div(1e3).round(2))) n.madd("Generator", ppl.index, carrier=ppl.carrier, bus=ppl.bus, - p_nom=ppl.p_nom, + p_nom_min=ppl.p_nom.where(ppl.carrier.isin(conventional_carriers), 0), + p_nom=ppl.p_nom.where(ppl.carrier.isin(conventional_carriers), 0), + p_nom_extendable=ppl.carrier.isin(extendable_carriers['Generator']), efficiency=ppl.efficiency, marginal_cost=ppl.marginal_cost, - capital_cost=0) - - logger.warning(f'Capital costs for conventional generators put to 0 EUR/MW.') + capital_cost=ppl.capital_cost, + build_year=ppl.datein.fillna(0).astype(int), + lifetime=(ppl.dateout - ppl.datein).fillna(9999).astype(int), + ) - for k,v in conventional_carriers["energy_availability_factors"].items(): - + for carrier in config: + # Generators with technology affected - idx = n.generators.query("carrier == @k").index + idx = n.generators.query("carrier == @carrier").index + factors = config[carrier].get("energy_availability_factors") if isinstance(v, float): # Single value affecting all generators of technology k indiscriminantely of country n.generators.loc[idx, "p_max_pu"] = v elif isinstance(v, dict): v = pd.Series(v) - # Values affecting generators of technology k country-specific # First map generator buses to countries; then map countries to p_max_pu - n.generators.loc[idx, "p_max_pu"] = n.generators.loc[idx]["bus"].replace(n.buses["country"]).replace(v) + n.generators.p_max_pu.update(n.generators.loc[idx].bus.map(v).dropna()) @@ -429,7 +434,7 @@ def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **con def attach_extendable_generators(n, costs, ppl, carriers): - + logger.warning("The function `attach_extendable_generators` is deprecated in v0.0.5.") _add_missing_carriers_from_costs(n, costs, carriers) for tech in carriers: @@ -476,24 +481,18 @@ def attach_extendable_generators(n, costs, ppl, carriers): -def attach_OPSD_renewables(n, techs): +def attach_OPSD_renewables(n, tech_map): - tech_map = {'Onshore': 'onwind', 'Offshore': 'offwind', 'Solar': 'solar'} - tech_map = {k: v for k, v in tech_map.items() if v in techs} - - if not tech_map: - return - - tech_string = ", ".join(tech_map.values()) - logger.info(f'Using OPSD renewable capacities for technologies {tech_string}.') + tech_string = ", ".join(sum(tech_map.values(), [])) + logger.info(f'Using OPSD renewable capacities for carriers {tech_string}.') df = pm.data.OPSD_VRE().powerplant.convert_country_to_alpha2() technology_b = ~df.Technology.isin(['Onshore', 'Offshore']) - df['Fueltype'] = df.Fueltype.where(technology_b, df.Technology) + df['Fueltype'] = df.Fueltype.where(technology_b, df.Technology).replace({"Solar": "PV"}) df = df.query('Fueltype in @tech_map').powerplant.convert_country_to_alpha2() - for fueltype, carrier_like in tech_map.items(): - gens = n.generators[lambda df: df.carrier.str.contains(carrier_like)] + for fueltype, carriers in tech_map.items(): + gens = n.generators[lambda df: df.carrier.isin(carriers)] buses = n.buses.loc[gens.bus.unique()] gens_per_bus = gens.groupby('bus').p_nom.count() @@ -505,32 +504,27 @@ def attach_OPSD_renewables(n, techs): n.generators.p_nom_min.update(gens.bus.map(caps).dropna()) - def estimate_renewable_capacities(n, config): - if not config["electricity"].get("estimate_renewable_capacities"): return - year = config["electricity"]["estimate_renewable_capacities"]["year"] tech_map = config["electricity"]["estimate_renewable_capacities"]["technology_mapping"] - tech_keys = list(tech_map.keys()) countries = config["countries"] expansion_limit = config["electricity"]["estimate_renewable_capacities"]["expansion_limit"] - if len(countries) == 0: return - if len(tech_map) == 0: return + if not len(countries) or not len(tech_map): return capacities = pm.data.IRENASTAT().powerplant.convert_country_to_alpha2() - capacities = capacities.query("Year == @year and Technology in @tech_keys and Country in @countries") + capacities = capacities.query("Year == @year and Technology in @tech_map and Country in @countries") capacities = capacities.groupby(["Technology", "Country"]).Capacity.sum() - logger.info(f"Heuristics applied to distribute renewable capacities [MW] " - f"{capacities.groupby('Country').sum()}") + logger.info(f"Heuristics applied to distribute renewable capacities [GW]: " + f"\n{capacities.groupby('Technology').sum().div(1e3).round(2)}") for ppm_technology, techs in tech_map.items(): tech_i = n.generators.query('carrier in @techs').index stats = capacities.loc[ppm_technology].reindex(countries, fill_value=0.) - country = n.generators[tech_i].bus.map(n.buses.country) + country = n.generators.bus[tech_i].map(n.buses.country) existent = n.generators.p_nom[tech_i].groupby(country).sum() missing = stats - existent dist = n.generators_t.p_max_pu.mean() * n.generators.p_nom_max @@ -571,29 +565,50 @@ if __name__ == "__main__": costs = load_costs(snakemake.input.tech_costs, snakemake.config['costs'], snakemake.config['electricity'], Nyears) ppl = load_powerplants(snakemake.input.powerplants) + + if "renewable_carriers" in snakemake.config['electricity']: + renewable_carriers = set(snakemake.config['renewable']) + else: + logger.warning("Key `renewable_carriers` not found in config under tag `electricity`, " + "falling back to carriers listed under `renewable`.") + renewable_carriers = snakemake.config['renewable'] + + extendable_carriers = snakemake.config['electricity']['extendable_carriers'] + if not (set(renewable_carriers) & set(extendable_carriers['Generator'])): + logger.warning(f"In future versions >= v0.0.6, extenable renewable carriers have to be " + "explicitely mentioned in `extendable_carriers`.") + + conventional_carriers = snakemake.config["electricity"]["conventional_carriers"] + attach_load(n, snakemake.input.regions, snakemake.input.load, snakemake.input.nuts3_shapes, snakemake.config['countries'], snakemake.config['load']['scaling_factor']) update_transmission_costs(n, costs, snakemake.config['lines']['length_factor']) - attach_conventional_generators(n, costs, ppl, snakemake.config["electricity"]["conventional_carriers"]) + attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers) - carriers = snakemake.config['renewable'] - attach_wind_and_solar(n, costs, snakemake.input, carriers, snakemake.config['lines']['length_factor']) + attach_wind_and_solar(n, costs, snakemake.input, renewable_carriers, extendable_carriers, snakemake.config['lines']['length_factor']) - if 'hydro' in snakemake.config['renewable']: - carriers = snakemake.config['renewable']['hydro'].pop('carriers', []) - attach_hydro(n, costs, ppl, snakemake.input.profile_hydro, snakemake.input.hydro_capacities, - carriers, **snakemake.config['renewable']['hydro']) + if 'hydro' in renewable_carriers: + conf = snakemake.config['renewable']['hydro'] + attach_hydro(n, costs, ppl, snakemake.input.profile_hydro, snakemake.input.hydro_capacities, + conf.pop('carriers', []), **conf) - carriers = snakemake.config['electricity']['extendable_carriers']['Generator'] - attach_extendable_generators(n, costs, ppl, carriers) + estimate_renewable_caps = snakemake.config['electricity'].get('estimate_renewable_capacities', {}) + if not isinstance(estimate_renewable_caps, dict): + logger.warning("The config entry `estimate_renewable_capacities` was changed to a dictionary, " + "please update your config yaml file accordingly.") + from_opsd = bool(snakemake.config["electricity"]["renewable_capacities_from_opsd"]) + estimate_renewable_caps = {"enable": True, "from_opsd": from_opsd} - techs = snakemake.config['electricity'].get('renewable_capacities_from_OPSD', []) - attach_OPSD_renewables(n, techs) + if estimate_renewable_caps["enable"]: + + if estimate_renewable_caps["from_opsd"]: + tech_map = snakemake.config["electricity"]["estimate_renewable_capacities"]["technology_mapping"] + attach_OPSD_renewables(n, tech_map) - estimate_renewable_capacities(n, snakemake.config) + estimate_renewable_capacities(n, snakemake.config) update_p_nom_max(n) From 26a56d1836fff183d5c996e76006b7a74e9f37e2 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 9 Jun 2022 20:33:13 +0200 Subject: [PATCH 34/61] update release notes --- doc/release_notes.rst | 76 ++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index f07176ad..2636ea31 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -7,13 +7,38 @@ Release Notes ########################################## -Energy Security Release (April 2022) -==================================== +Upcoming Release +================ -**New Features and Changes** +* Add an efficiency factor of 88.55% to offshore wind capacity factors + as a proxy for wake losses. More rigorous modelling is `planned `_ + [`#277 `_]. + +* The default deployment density of AC- and DC-connected offshore wind capacity is reduced from 3 MW/sqkm + to a more conservative estimate of 2 MW/sqkm [`#280 `_]. + +* Following discussion in `#285 `_ we have disabled the + correction factor for solar PV capacity factors by default while satellite data is used. + A correction factor of 0.854337 is recommended if reanalysis data like ERA5 is used. + +* Resource definitions for memory usage now follow [Snakemake standard resource definition](https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#standard-resources) ```mem_mb`` rather than ``mem``. + +* Network building is made deterministic by supplying a fixed random state to network clustering routines. + +* New network topology extracted from the ENTSO-E interactive map. + +* The unused argument ``simple_hvdc_costs`` in :mod:`add_electricity` was removed. + +* Iterative solving with impedance updates is skipped if there are no expandable lines. + +* Switch from Germany to Belgium for continuous integration and tutorial to save resources. + +* Use updated SARAH-2 and ERA5 cutouts with slightly wider scope to east and additional variables. * Added existing renewable capacities for all countries based on IRENA statistics (IRENASTAT) using new ``powerplantmatching`` version: + * The estimation is endabled by setting ``enable`` to ``True``. * Configuration of reference year for capacities can be configured (default: ``2020``) + * The list of renewables provided by the OPSD database can be used as a basis, using the tag ``from_opsd: True``. This adds the renewables from the database and fills up the missing capacities with the heuristic distribution. * Uniform expansion limit of renewable build-up based on existing capacities can be configured using ``expansion_limit`` option (default: ``false``; limited to determined renewable potentials) * Distribution of country-level capacities proportional to maximum annual energy yield for each bus region @@ -22,6 +47,8 @@ Energy Security Release (April 2022) * old: ``estimate_renewable_capacities_from_capacity_stats`` * new: ``estimate_renewable_capacities`` +* The config key ``renewable_capacities_from_OPSD`` is deprecated and was moved under the section, ``estimate_renewable_capacities``. To enable it set ``from_opsd`` to `True`. + * Add operational reserve margin constraint analogous to `GenX implementation `_. Can be activated with config setting ``electricity: operational_reserve:``. @@ -31,11 +58,22 @@ Energy Security Release (April 2022) * Implement country-specific EAFs for nuclear power plants based on IAEA 2018-2020 reported country averages. * The powerplants that have been shut down before 2021 are filtered out. - -**Bugs and Compatibility** * ``powerplantmatching>=0.5.1`` is now required for ``IRENASTATS``. +* The interpretation of ``extendable_carriers`` in the config was changed that all carriers that should be extendable have to be listed here. Before, renewable carriers were always set to be extendable. For backwards compatibility, the workflow is looking at both the listed carriers under the ``renewable`` key and the ``extendable`` key. But in the future, all of them have to be listed under ``extendable_carriers``. + +* It is now possible to set conventional power plants as extendable by adding them to the list of extendable ``Generator`` carriers in the config. + +* By having carriers in the list of ``extendable_carriers`` but not in the list of ``conventional_carriers``, the corresponding conventional power plants are set extendable without a lower capacity bound of today's capacities. + +* Now, conventional carriers have an assigned capital cost by default. + +* The ``build_year`` and ``lifetime`` column are now defined for conventional power plants. + +* A new section ``conventional`` was added to the config file. This section contains configurations for conventional carriers. Using the ``energy_availibility_factor`` key, the ``p_max_pu`` values for conventional power plants can be defined. + + Synchronisation Release - Ukraine and Moldova (17th March 2022) =============================================================== @@ -71,34 +109,6 @@ This release is not on the ``master`` branch. It can be used with git checkout synchronisation-release -Upcoming Release -================ - -* Add an efficiency factor of 88.55% to offshore wind capacity factors - as a proxy for wake losses. More rigorous modelling is `planned `_ - [`#277 `_]. - -* The default deployment density of AC- and DC-connected offshore wind capacity is reduced from 3 MW/sqkm - to a more conservative estimate of 2 MW/sqkm [`#280 `_]. - -* Following discussion in `#285 `_ we have disabled the - correction factor for solar PV capacity factors by default while satellite data is used. - A correction factor of 0.854337 is recommended if reanalysis data like ERA5 is used. - -* Resource definitions for memory usage now follow [Snakemake standard resource definition](https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#standard-resources) ```mem_mb`` rather than ``mem``. - -* Network building is made deterministic by supplying a fixed random state to network clustering routines. - -* New network topology extracted from the ENTSO-E interactive map. - -* The unused argument ``simple_hvdc_costs`` in :mod:`add_electricity` was removed. - -* Iterative solving with impedance updates is skipped if there are no expandable lines. - -* Switch from Germany to Belgium for continuous integration and tutorial to save resources. - -* Use updated SARAH-2 and ERA5 cutouts with slightly wider scope to east and additional variables. - PyPSA-Eur 0.4.0 (22th September 2021) ===================================== From f45803ff10d6e1afec101400c8e98215611515c4 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 9 Jun 2022 23:40:32 +0200 Subject: [PATCH 35/61] add_electricity: fix missing backwards compat and warnings --- scripts/add_electricity.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 24027162..880dcaad 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -25,7 +25,7 @@ Relevant Settings co2limit: extendable_carriers: estimate_renewable_capacities: - + load: scaling_factor: @@ -580,14 +580,16 @@ if __name__ == "__main__": if "renewable_carriers" in snakemake.config['electricity']: renewable_carriers = set(snakemake.config['renewable']) else: - logger.warning("Key `renewable_carriers` not found in config under tag `electricity`, " - "falling back to carriers listed under `renewable`.") + logger.warning("Missing key `renewable_carriers` under config entry `electricity`. " + "In future versions, this will raise an error. " + "Falling back to carriers listed under `renewable`.") renewable_carriers = snakemake.config['renewable'] extendable_carriers = snakemake.config['electricity']['extendable_carriers'] if not (set(renewable_carriers) & set(extendable_carriers['Generator'])): - logger.warning(f"In future versions >= v0.0.6, extenable renewable carriers have to be " - "explicitely mentioned in `extendable_carriers`.") + logger.warning("No renewables found in config entry `extendable_carriers`. " + "In future versions, these have to be explicitely listed. " + "Falling back to all renewables.") conventional_carriers = snakemake.config["electricity"]["conventional_carriers"] @@ -606,19 +608,26 @@ if __name__ == "__main__": attach_hydro(n, costs, ppl, snakemake.input.profile_hydro, snakemake.input.hydro_capacities, conf.pop('carriers', []), **conf) - estimate_renewable_caps = snakemake.config['electricity'].get('estimate_renewable_capacities', {}) - if not isinstance(estimate_renewable_caps, dict): - logger.warning("The config entry `estimate_renewable_capacities` was changed to a dictionary, " - "please update your config yaml file accordingly.") + if "estimate_renewable_capacities" not in snakemake.config['electricity']: + logger.warning("Missing key `estimate_renewable_capacities` under config entry `electricity`." + "In future versions, this will raise an error. ") + estimate_renewable_caps = {'enable': False} + if "enable" not in estimate_renewable_caps: + logger.warning("Missing key `enable` under config entry `estimate_renewable_capacities`." + "In future versions, this will raise an error. Falling back to False.") + estimate_renewable_caps = {'enable': False} + if "from_opsd" not in estimate_renewable_caps: + logger.warning("Missing key `from_opsd` under config entry `estimate_renewable_capacities`." + "In future versions, this will raise an error. " + "Falling back to whether `renewable_capacities_from_opsd` is non-empty.") from_opsd = bool(snakemake.config["electricity"]["renewable_capacities_from_opsd"]) - estimate_renewable_caps = {"enable": True, "from_opsd": from_opsd} + estimate_renewable_caps['from_opsd'] = from_opsd + - if estimate_renewable_caps["enable"]: - + if estimate_renewable_caps["enable"]: if estimate_renewable_caps["from_opsd"]: tech_map = snakemake.config["electricity"]["estimate_renewable_capacities"]["technology_mapping"] attach_OPSD_renewables(n, tech_map) - estimate_renewable_capacities(n, snakemake.config) update_p_nom_max(n) From 8cbe4e4f9dfdcbb3a5d92104e0a76497138469aa Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 10 Jun 2022 00:33:09 +0200 Subject: [PATCH 36/61] update release notes and doc --- doc/configtables/electricity.csv | 8 ++--- doc/configuration.rst | 3 -- doc/release_notes.rst | 52 +++++++++++++++++--------------- 3 files changed, 31 insertions(+), 32 deletions(-) diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index aef35350..3ec26106 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -4,7 +4,7 @@ co2limit,:math:`t_{CO_2-eq}/a`,float,Cap on total annual system carbon dioxide e co2base,:math:`t_{CO_2-eq}/a`,float,Reference value of total annual system carbon dioxide emissions if relative emission reduction target is specified in ``{opts}`` wildcard. agg_p_nom_limits,file,path,Reference to ``.csv`` file specifying per carrier generator nominal capacity constraints for individual countries if ``'CCL'`` is in ``{opts}`` wildcard. Defaults to ``data/agg_p_nom_minmax.csv``. extendable_carriers,,, --- Generator,--,"Any subset of {'OCGT','CCGT'}",Places extendable conventional power plants (OCGT and/or CCGT) where gas power plants are located today without capacity limits. +-- Generator,--,"Any extendable carrier",Defines existing or non-existing conventional and renewable power plants to be extendable during the optimization. Conventional generators can only be built/expanded where already existent today. If a listed conventional carrier is not included in the ``conventional_carriers`` list, the lower limit of the capacity expansion is set to 0. -- StorageUnit,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity. -- Store,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity. -- Link,--,Any subset of {'H2 pipeline'},Adds extendable links (H2 pipelines only) at every connection where there are lines or HVDC links without capacity limits and with zero initial capacity. Hydrogen pipelines require hydrogen storage to be modelled as ``Store``. @@ -13,7 +13,7 @@ max_hours,,, -- H2,h,float,Maximum state of charge capacity of the hydrogen storage in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_. powerplants_filter,--,"use `pandas.query `_ strings here, e.g. Country not in ['Germany']",Filter query for the default powerplant database. custom_powerplants,--,"use `pandas.query `_ strings here, e.g. Country in ['Germany']",Filter query for the custom powerplant database. -conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}",List of conventional power plants to include in the model from ``resources/powerplants.csv``. -renewable_capacities_from_OPSD,,"[solar, onwind, offwind]",List of carriers (offwind-ac and offwind-dc are included in offwind) whose capacities 'p_nom' are aligned to the `OPSD renewable power plant list `_ -estimate_renewable_capacities_from_capacitiy_stats,,, +conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}",List of conventional power plants to include in the model from ``resources/powerplants.csv``. If an included carrier is also listed in `extendable_carriers`, the capacity is taken as a lower bound. +renewable_carriers,--,"Any subset of {solar, offwind-ac, offwind-dc, hydro}",List of renewable generators to include in the model. +estimate_renewable_capacities,,, "-- Fueltype [ppm], e.g. Wind",,"list of fueltypes strings in PyPSA-Eur, e.g. [onwind, offwind-ac, offwind-dc]",converts ppm Fueltype to PyPSA-EUR Fueltype diff --git a/doc/configuration.rst b/doc/configuration.rst index a448f817..67d25228 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -91,9 +91,6 @@ Specifies the temporal range to build an energy system model for as arguments to :widths: 25,7,22,30 :file: configtables/electricity.csv -.. warning:: - Carriers in ``conventional_carriers`` must not also be in ``extendable_carriers``. - .. _atlite_cf: ``atlite`` diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 5ceea76d..1825c65c 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -36,42 +36,44 @@ Upcoming Release * Use updated SARAH-2 and ERA5 cutouts with slightly wider scope to east and additional variables. * Added existing renewable capacities for all countries based on IRENA statistics (IRENASTAT) using new ``powerplantmatching`` version: - * The estimation is endabled by setting ``enable`` to ``True``. - * Configuration of reference year for capacities can be configured (default: ``2020``) - * The list of renewables provided by the OPSD database can be used as a basis, using the tag ``from_opsd: True``. This adds the renewables from the database and fills up the missing capacities with the heuristic distribution. - * Uniform expansion limit of renewable build-up based on existing capacities can be configured using ``expansion_limit`` option - (default: ``false``; limited to determined renewable potentials) - * Distribution of country-level capacities proportional to maximum annual energy yield for each bus region - * This functionality was previously using OPSD data. - * The corresponding ``config`` entries changed, cf. ``config.default.yaml``: - * old: ``estimate_renewable_capacities_from_capacity_stats`` - * new: ``estimate_renewable_capacities`` + * The corresponding ``config`` entries changed, cf. ``config.default.yaml``: + * old: ``estimate_renewable_capacities_from_capacity_stats`` + * new: ``estimate_renewable_capacities`` + * The estimation is endabled by setting the subkey ``enable`` to ``True``. + * Configuration of reference year for capacities can be configured (default: ``2020``) + * The list of renewables provided by the OPSD database can be used as a basis, using the tag ``from_opsd: True``. This adds the renewables from the database and fills up the missing capacities with the heuristic distribution. + * Uniform expansion limit of renewable build-up based on existing capacities can be configured using ``expansion_limit`` option + (default: ``false``; limited to determined renewable potentials) + * Distribution of country-level capacities proportional to maximum annual energy yield for each bus region -* The config key ``renewable_capacities_from_OPSD`` is deprecated and was moved under the section, ``estimate_renewable_capacities``. To enable it set ``from_opsd`` to `True`. +* The config key ``renewable_capacities_from_OPSD`` is deprecated and was moved under the section, ``estimate_renewable_capacities``. To enable it, set ``from_opsd`` to `True`. -* Add operational reserve margin constraint analogous to `GenX implementation `_. - Can be activated with config setting ``electricity: operational_reserve:``. +* Add operational reserve margin constraint analogous to `GenX implementation `_. + Can be activated with config setting ``electricity: operational_reserve:``. -* Add function to add global constraint on use of gas in :mod:`prepare_network`. This can be activated with `electricity: gaslimit:` given in MWh. +* Add function to add global constraint on use of gas in :mod:`prepare_network`. This can be activated with `electricity: gaslimit:` given in MWh. -* Add configuration option to implement Energy Availability Factors (EAFs) for conventional generation technologies. -* Implement country-specific EAFs for nuclear power plants based on IAEA 2018-2020 reported country averages. +* Add configuration option to implement Energy Availability Factors (EAFs) for conventional generation technologies. -* The powerplants that have been shut down before 2021 are filtered out. - -* ``powerplantmatching>=0.5.1`` is now required for ``IRENASTATS``. +* A new section ``conventional`` was added to the config file. This section contains configurations for conventional carriers. -* The interpretation of ``extendable_carriers`` in the config was changed that all carriers that should be extendable have to be listed here. Before, renewable carriers were always set to be extendable. For backwards compatibility, the workflow is looking at both the listed carriers under the ``renewable`` key and the ``extendable`` key. But in the future, all of them have to be listed under ``extendable_carriers``. +* Implement country-specific EAFs for nuclear power plants based on IAEA 2018-2020 reported country averages. These are specified under the ``energy_availibility_factor`` key in the config entry ``conventional`` and specify the static `p_max_pu` values. -* It is now possible to set conventional power plants as extendable by adding them to the list of extendable ``Generator`` carriers in the config. +* The powerplants that have been shut down before 2021 are filtered out. -* By having carriers in the list of ``extendable_carriers`` but not in the list of ``conventional_carriers``, the corresponding conventional power plants are set extendable without a lower capacity bound of today's capacities. +* ``powerplantmatching>=0.5.1`` is now required for ``IRENASTATS``. -* Now, conventional carriers have an assigned capital cost by default. +* The inclusion of renewable carriers is now specified in the config entry ``renewable_carriers``. Before this was done by commenting/uncommenting sub-sections in the `renewable` config section. -* The ``build_year`` and ``lifetime`` column are now defined for conventional power plants. +* Now, all carriers that should be extendable have to be listed in the config entry ``extendable_carriers``. Before, renewable carriers were always set to be extendable. For backwards compatibility, the workflow is still looking at the listed carriers under the ``renewable`` key. In the future, all of them have to be listed under ``extendable_carriers``. -* A new section ``conventional`` was added to the config file. This section contains configurations for conventional carriers. Using the ``energy_availibility_factor`` key, the ``p_max_pu`` values for conventional power plants can be defined. +* It is now possible to set conventional power plants as extendable by adding them to the list of extendable ``Generator`` carriers in the config. + +* Listing conventional carriers in ``extendable_carriers`` but not in ``conventional_carriers``, sets the corresponding conventional power plants as extendable without a lower capacity bound of today's capacities. + +* Now, conventional carriers have an assigned capital cost by default. + +* The ``build_year`` and ``lifetime`` column are now defined for conventional power plants. * Fix crs bug. Change crs 4236 to 4326. From c68aa028ffff8d83a4ea3fa0dcefcf8e256aa58a Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 10 Jun 2022 00:36:07 +0200 Subject: [PATCH 37/61] add_electricity: fix missing config key --- scripts/add_electricity.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 880dcaad..bd17dbe5 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -609,18 +609,22 @@ if __name__ == "__main__": conf.pop('carriers', []), **conf) if "estimate_renewable_capacities" not in snakemake.config['electricity']: - logger.warning("Missing key `estimate_renewable_capacities` under config entry `electricity`." - "In future versions, this will raise an error. ") - estimate_renewable_caps = {'enable': False} + logger.warning("Missing key `estimate_renewable_capacities` under config entry `electricity`. " + "In future versions, this will raise an error. " + "Falling back to whether ``estimate_renewable_capacities_from_capacity_stats`` is in the config.") + if "estimate_renewable_capacities_from_capacity_stats" in snakemake.config['electricity']: + estimate_renewable_caps = {'enable': True, **snakemake.config['electricity']["estimate_renewable_capacities_from_capacity_stats"]} + else: + estimate_renewable_caps = {'enable': False} if "enable" not in estimate_renewable_caps: - logger.warning("Missing key `enable` under config entry `estimate_renewable_capacities`." + logger.warning("Missing key `enable` under config entry `estimate_renewable_capacities`. " "In future versions, this will raise an error. Falling back to False.") estimate_renewable_caps = {'enable': False} if "from_opsd" not in estimate_renewable_caps: - logger.warning("Missing key `from_opsd` under config entry `estimate_renewable_capacities`." + logger.warning("Missing key `from_opsd` under config entry `estimate_renewable_capacities`. " "In future versions, this will raise an error. " "Falling back to whether `renewable_capacities_from_opsd` is non-empty.") - from_opsd = bool(snakemake.config["electricity"]["renewable_capacities_from_opsd"]) + from_opsd = bool(snakemake.config["electricity"].get("renewable_capacities_from_opsd", False)) estimate_renewable_caps['from_opsd'] = from_opsd From 13992125bdaf3d8b7190b6c396f7ccaa2a3a75a7 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 10 Jun 2022 00:54:54 +0200 Subject: [PATCH 38/61] cluster_network: adjust generator strategy for new columns --- scripts/add_electricity.py | 2 +- scripts/cluster_network.py | 6 ++++-- scripts/simplify_network.py | 15 +++++++++++++-- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index bd17dbe5..87516a83 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -309,7 +309,7 @@ def attach_conventional_generators(n, costs, ppl, conventional_carriers, extenda ppl = (ppl.query('carrier in @carriers').join(costs, on='carrier', rsuffix='_r') .rename(index=lambda s: 'C' + str(s))) - ppl.efficiency.update(ppl.efficiency_r.dropna()) + ppl["efficiency"] = ppl.efficiency.fillna(ppl.efficiency_r) logger.info('Adding {} generators with capacities [GW] \n{}' .format(len(ppl), ppl.groupby('carrier').p_nom.sum().div(1e3).round(2))) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 642db4da..a3ce5c52 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -281,7 +281,9 @@ def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carr aggregate_generators_carriers=aggregate_carriers, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=line_length_factor, - generator_strategies={'p_nom_max': p_nom_max_strategy, 'p_nom_min': pd.Series.sum}, + generator_strategies={'p_nom_max': p_nom_max_strategy, 'p_nom_min': pd.Series.sum, + 'build_year': lambda x: 0, 'lifetime': lambda x: np.inf, + 'efficiency': np.mean}, scale_link_capital_costs=False) if not n.links.empty: @@ -342,7 +344,7 @@ if __name__ == "__main__": if snakemake.wildcards.clusters.endswith('m'): n_clusters = int(snakemake.wildcards.clusters[:-1]) - aggregate_carriers = pd.Index(n.generators.carrier.unique()).difference(renewable_carriers) + aggregate_carriers = None elif snakemake.wildcards.clusters == 'all': n_clusters = len(n.buses) aggregate_carriers = None # All diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 287dfe32..5f9aec6c 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -200,7 +200,14 @@ def _aggregate_and_move_components(n, busmap, connection_costs_to_bus, output, a _adjust_capital_costs_using_connection_costs(n, connection_costs_to_bus, output) - generators, generators_pnl = aggregategenerators(n, busmap, custom_strategies={'p_nom_min': np.sum}) + strategies = { + 'p_nom_min': np.sum, + 'p_nom_max': 'sum', + 'build_year': lambda x: 0, + 'lifetime': lambda x: np.inf, + 'efficiency': np.mean + } + generators, generators_pnl = aggregategenerators(n, busmap, custom_strategies=strategies) replace_components(n, "Generator", generators, generators_pnl) for one_port in aggregate_one_ports: @@ -351,7 +358,11 @@ def aggregate_to_substations(n, buses_i=None): aggregate_generators_carriers=None, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=1.0, - generator_strategies={'p_nom_max': 'sum'}, + generator_strategies={'p_nom_max': 'sum', + 'build_year': lambda x: 0, + 'lifetime': lambda x: np.inf, + 'efficiency': np.mean + }, scale_link_capital_costs=False) return clustering.network, busmap From 1c0975181e17b324b35e5b55986039502d0e73fc Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 10 Jun 2022 11:23:35 +0200 Subject: [PATCH 39/61] add_electricity: fix missing config key --- scripts/add_electricity.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 87516a83..ba1bf9bf 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -616,6 +616,8 @@ if __name__ == "__main__": estimate_renewable_caps = {'enable': True, **snakemake.config['electricity']["estimate_renewable_capacities_from_capacity_stats"]} else: estimate_renewable_caps = {'enable': False} + else: + estimate_renewable_caps = snakemake.config['electricity']["estimate_renewable_capacities"] if "enable" not in estimate_renewable_caps: logger.warning("Missing key `enable` under config entry `estimate_renewable_capacities`. " "In future versions, this will raise an error. Falling back to False.") From f6b7317d043b8d53ba779727da273bd1afd5d81b Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Thu, 23 Jun 2022 15:14:38 +0200 Subject: [PATCH 40/61] Update config.tutorial.yaml Co-authored-by: Martha Frysztacki --- config.tutorial.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 3238329c..568119cd 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -9,7 +9,6 @@ logging: level: INFO format: '%(levelname)s:%(name)s:%(message)s' -results_dir: results/your-run-name scenario: simpl: [''] From db78f9cd2adec74c75ef8c286b0cf2699a823997 Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Thu, 23 Jun 2022 15:54:22 +0200 Subject: [PATCH 41/61] Update scripts/add_electricity.py Co-authored-by: Martha Frysztacki --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index ba1bf9bf..4e4fdecd 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -599,7 +599,7 @@ if __name__ == "__main__": update_transmission_costs(n, costs, snakemake.config['lines']['length_factor']) - attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers) + attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers, **snakemake.config["conventional"]) attach_wind_and_solar(n, costs, snakemake.input, renewable_carriers, extendable_carriers, snakemake.config['lines']['length_factor']) From 441d7d56f9f25b0b187008931c3c615b1d599942 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 23 Jun 2022 16:04:49 +0200 Subject: [PATCH 42/61] fix eafs and conventional setttings --- config.default.yaml | 23 +++-------------------- scripts/add_electricity.py | 17 +++++++++-------- scripts/solve_network.py | 4 ++++ 3 files changed, 16 insertions(+), 28 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index fc09dfd6..30f1d2b6 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -43,14 +43,14 @@ electricity: agg_p_nom_limits: data/agg_p_nom_minmax.csv operational_reserve: # like https://genxproject.github.io/GenX/dev/core/#Reserves - activate: true + activate: false epsilon_load: 0.02 # share of total load epsilon_vres: 0.02 # share of total renewable supply contingency: 4000 # fixed capacity in MW max_hours: battery: 6 - H2: 168 + H2: 168 extendable_carriers: Generator: [solar, onwind, offwind-ac, offwind-dc, OCGT] @@ -189,24 +189,7 @@ renewable: conventional: nuclear: - energy_availability_factors: - # From IAEA - # https://pris.iaea.org/PRIS/WorldStatistics/ThreeYrsEnergyAvailabilityFactor.aspx (2022-04-08) - BE: 0.65 - BG: 0.89 - CZ: 0.82 - FI: 0.92 - FR: 0.70 - DE: 0.88 - HU: 0.90 - NL: 0.86 - RO: 0.92 - SK: 0.89 - SI: 0.94 - ES: 0.89 - SE: 0.82 - CH: 0.86 - GB: 0.67 + energy_availability_factors: "data/nuclear-eafs.csv" # float of file name lines: types: diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 4e4fdecd..c37a791d 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -302,7 +302,7 @@ def attach_wind_and_solar(n, costs, input_profiles, technologies, extendable_car p_max_pu=ds['profile'].transpose('time', 'bus').to_pandas()) -def attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers, **config): +def attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers, conventional_config): carriers = set(conventional_carriers) | set(extendable_carriers['Generator']) _add_missing_carriers_from_costs(n, costs, carriers) @@ -327,20 +327,21 @@ def attach_conventional_generators(n, costs, ppl, conventional_carriers, extenda lifetime=(ppl.dateout - ppl.datein).fillna(9999).astype(int), ) - for carrier in config: + for carrier in conventional_config: # Generators with technology affected idx = n.generators.query("carrier == @carrier").index - factors = config[carrier].get("energy_availability_factors") + factors = conventional_config[carrier].get("energy_availability_factors") - if isinstance(v, float): + if isinstance(factors, float): # Single value affecting all generators of technology k indiscriminantely of country - n.generators.loc[idx, "p_max_pu"] = v - elif isinstance(v, dict): - v = pd.Series(v) + n.generators.loc[idx, "p_max_pu"] = factors + elif isinstance(factors, str): + factors = pd.read_file(factors, index_col=0) # Values affecting generators of technology k country-specific # First map generator buses to countries; then map countries to p_max_pu - n.generators.p_max_pu.update(n.generators.loc[idx].bus.map(v).dropna()) + bus_factors = n.buses.country.map(factors) + n.generators.p_max_pu.update(n.generators.loc[idx].bus.map(bus_factors).dropna()) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index a5ebfe6e..06296723 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -271,6 +271,10 @@ def update_capacity_constraint(n): def add_operational_reserve_margin(n, sns, config): + """ + Build reserve margin constraints based on the formulation given in + https://genxproject.github.io/GenX/dev/core/#Reserves. + """ define_variables(n, 0, np.inf, 'Generator', 'r', axes=[sns, n.generators.index]) From 95e8a9534a3410fc75bd76bc7d8b47b97e12fe1e Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 23 Jun 2022 16:07:33 +0200 Subject: [PATCH 43/61] env: remove duplicated ppm dependency --- envs/environment.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 0c636edc..73c14fa5 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -57,4 +57,3 @@ dependencies: - pip: - vresutils>=0.3.1 - tsam>=1.1.0 - - powerplantmatching>=0.5.3 From d6930b878afab2084df172269a6778f288fdc2cc Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Thu, 23 Jun 2022 16:14:20 +0200 Subject: [PATCH 44/61] Update scripts/cluster_network.py Co-authored-by: Martha Frysztacki --- scripts/cluster_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 5166f37a..833614ab 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -352,7 +352,7 @@ if __name__ == "__main__": if snakemake.wildcards.clusters.endswith('m'): n_clusters = int(snakemake.wildcards.clusters[:-1]) - aggregate_carriers = None + aggregate_carriers = snakemake.config["electricity"].get("conventional_carriers") elif snakemake.wildcards.clusters == 'all': n_clusters = len(n.buses) aggregate_carriers = None # All From 348b14b052fcd5744494b8aa87cba62bf467bf62 Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Thu, 23 Jun 2022 16:15:37 +0200 Subject: [PATCH 45/61] Update scripts/build_hydro_profile.py Co-authored-by: Martha Frysztacki --- scripts/build_hydro_profile.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 4add4c85..bfba40b2 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -78,8 +78,6 @@ def get_eia_annual_hydro_generation(fn, countries): df.loc["Germany"] = df.filter(like='Germany', axis=0).sum() df.loc["Serbia"] += df.loc["Kosovo"] - df = df.loc[~df.index.str.contains('Former')] - df.drop(["Europe", "Germany, West", "Germany, East"], inplace=True) df.index = cc.convert(df.index, to='iso2') df.index.name = 'countries' From 2e212fb436053af0e2b654b6998a2f20ac763ea1 Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Thu, 23 Jun 2022 16:16:36 +0200 Subject: [PATCH 46/61] Update test/config.test1.yaml Co-authored-by: Martha Frysztacki --- test/config.test1.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 600a3aac..18c01ad2 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -8,7 +8,6 @@ logging: level: INFO format: '%(levelname)s:%(name)s:%(message)s' -results_dir: results/your-run-name scenario: simpl: [''] From 2c5643a5f8edf85130ca88c848e16f628024ae52 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 23 Jun 2022 16:36:22 +0200 Subject: [PATCH 47/61] add_electricity: fix conventional config --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index c37a791d..f2dfdfef 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -600,7 +600,7 @@ if __name__ == "__main__": update_transmission_costs(n, costs, snakemake.config['lines']['length_factor']) - attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers, **snakemake.config["conventional"]) + attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers, snakemake.config.get("conventional", {})) attach_wind_and_solar(n, costs, snakemake.input, renewable_carriers, extendable_carriers, snakemake.config['lines']['length_factor']) From b581e7afc6a6ed0268315c53f9b86c4e11229b2b Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 23 Jun 2022 21:19:41 +0200 Subject: [PATCH 48/61] build_hydro_profiles: revert changes --- scripts/build_hydro_profile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index bfba40b2..4add4c85 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -78,6 +78,8 @@ def get_eia_annual_hydro_generation(fn, countries): df.loc["Germany"] = df.filter(like='Germany', axis=0).sum() df.loc["Serbia"] += df.loc["Kosovo"] + df = df.loc[~df.index.str.contains('Former')] + df.drop(["Europe", "Germany, West", "Germany, East"], inplace=True) df.index = cc.convert(df.index, to='iso2') df.index.name = 'countries' From 17d7403f20fd5d7e279f33feae9d777a04da371e Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Fri, 24 Jun 2022 15:34:26 +0200 Subject: [PATCH 49/61] build_powerplants: apply suggestions from code review Co-authored-by: Martha Frysztacki --- doc/configtables/electricity.csv | 2 +- scripts/build_powerplants.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index 3ec26106..5abae842 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -14,6 +14,6 @@ max_hours,,, powerplants_filter,--,"use `pandas.query `_ strings here, e.g. Country not in ['Germany']",Filter query for the default powerplant database. custom_powerplants,--,"use `pandas.query `_ strings here, e.g. Country in ['Germany']",Filter query for the custom powerplant database. conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}",List of conventional power plants to include in the model from ``resources/powerplants.csv``. If an included carrier is also listed in `extendable_carriers`, the capacity is taken as a lower bound. -renewable_carriers,--,"Any subset of {solar, offwind-ac, offwind-dc, hydro}",List of renewable generators to include in the model. +renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, hydro}",List of renewable generators to include in the model. estimate_renewable_capacities,,, "-- Fueltype [ppm], e.g. Wind",,"list of fueltypes strings in PyPSA-Eur, e.g. [onwind, offwind-ac, offwind-dc]",converts ppm Fueltype to PyPSA-EUR Fueltype diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index c1ee4127..a5dbf57b 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -131,7 +131,7 @@ if __name__ == "__main__": custom_ppl_query = snakemake.config['electricity']['custom_powerplants'] ppl = add_custom_powerplants(ppl, snakemake.input.custom_powerplants, custom_ppl_query) - countries_wo_ppl = [c for c in countries if c not in ppl.Country.unique()] + countries_wo_ppl = set(countries)-set(ppl.Country.unique()) if countries_wo_ppl: logging.warning(f"No powerplants known in: {', '.join(countries_wo_ppl)}") From 3dbd8d1492b164f3f2f4c126e63cf35837af007a Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 24 Jun 2022 18:31:39 +0200 Subject: [PATCH 50/61] add_electricity: fix read_csv --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index f2dfdfef..9a99d314 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -337,7 +337,7 @@ def attach_conventional_generators(n, costs, ppl, conventional_carriers, extenda # Single value affecting all generators of technology k indiscriminantely of country n.generators.loc[idx, "p_max_pu"] = factors elif isinstance(factors, str): - factors = pd.read_file(factors, index_col=0) + factors = pd.read_csv(factors, index_col=0)['factor'] # Values affecting generators of technology k country-specific # First map generator buses to countries; then map countries to p_max_pu bus_factors = n.buses.country.map(factors) From a1ee747dc612e301477d09c2c50987a49587b231 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 24 Jun 2022 18:37:43 +0200 Subject: [PATCH 51/61] data: add nuclear_eafs --- data/nuclear_eafs.csv | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 data/nuclear_eafs.csv diff --git a/data/nuclear_eafs.csv b/data/nuclear_eafs.csv new file mode 100644 index 00000000..06b5f684 --- /dev/null +++ b/data/nuclear_eafs.csv @@ -0,0 +1,16 @@ +country,factor +BE,0.65 +BG,0.89 +CZ,0.82 +FI,0.92 +FR,0.70 +DE,0.88 +HU,0.90 +NL,0.86 +RO,0.92 +SK,0.89 +SI,0.94 +ES,0.89 +SE,0.82 +CH,0.86 +GB,0.67 \ No newline at end of file From b37c1d98d32274708f8bff8bc3e53bc9f9e283fb Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 24 Jun 2022 18:44:03 +0200 Subject: [PATCH 52/61] update release notes --- doc/release_notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 1825c65c..337f09e9 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -57,7 +57,7 @@ Upcoming Release * A new section ``conventional`` was added to the config file. This section contains configurations for conventional carriers. -* Implement country-specific EAFs for nuclear power plants based on IAEA 2018-2020 reported country averages. These are specified under the ``energy_availibility_factor`` key in the config entry ``conventional`` and specify the static `p_max_pu` values. +* Implement country-specific EAFs for nuclear power plants based on IAEA 2018-2020 reported country averages. These are specified `data/nuclear_eafs.csv` and translate to static `p_max_pu` values. * The powerplants that have been shut down before 2021 are filtered out. From 8349e85252d2e296d9ba45f4bbfd60e621a68599 Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Mon, 27 Jun 2022 11:48:45 +0200 Subject: [PATCH 53/61] Apply suggestions from code review Co-authored-by: Fabian Neumann --- doc/release_notes.rst | 8 +++++--- scripts/add_electricity.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 300bef88..57c0a2f0 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -21,7 +21,7 @@ Upcoming Release correction factor for solar PV capacity factors by default while satellite data is used. A correction factor of 0.854337 is recommended if reanalysis data like ERA5 is used. -* Resource definitions for memory usage now follow [Snakemake standard resource definition](https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#standard-resources) ```mem_mb`` rather than ``mem``. +* Resource definitions for memory usage now follow `Snakemake standard resource definition `_ ``mem_mb`` rather than ``mem``. * Network building is made deterministic by supplying a fixed random state to network clustering routines. @@ -51,13 +51,13 @@ Upcoming Release * Add operational reserve margin constraint analogous to `GenX implementation `_. Can be activated with config setting ``electricity: operational_reserve:``. -* Add function to add global constraint on use of gas in :mod:`prepare_network`. This can be activated with `electricity: gaslimit:` given in MWh. +* Add function to add global constraint on use of gas in :mod:`prepare_network`. This can be activated by including the keyword ``CH4L`` in the ``{opts}`` wildcard which enforces the limit set in ``electricity: gaslimit:`` given in MWh thermal. Alternatively, it is possible to append a number in the `{opts}` wildcard, e.g. `CH4L200` which limits the gas use to 200 TWh thermal. * Add configuration option to implement Energy Availability Factors (EAFs) for conventional generation technologies. * A new section ``conventional`` was added to the config file. This section contains configurations for conventional carriers. -* Implement country-specific EAFs for nuclear power plants based on IAEA 2018-2020 reported country averages. These are specified `data/nuclear_eafs.csv` and translate to static `p_max_pu` values. +* Implement country-specific EAFs for nuclear power plants based on IAEA 2018-2020 reported country averages. These are specified ``data/nuclear_eafs.csv`` and translate to static ``p_max_pu`` values. * The powerplants that have been shut down before 2021 are filtered out. @@ -84,7 +84,9 @@ Upcoming Release * Cache data and cutouts folders. This cache will be updated weekly. * Add rule to automatically retrieve Natura2000 natural protection areas. Switch of file format to GPKG. +* Add option to set CO2 emission prices through `{opts}` wildcard: `Ep`, e.g. `Ep180`, will set the EUR/tCO2 price. +* Add option to alter marginal costs of a carrier through `{opts}` wildcard: `+m`, e.g. `gas+m2.5`, will multiply the default marginal cost for gas by factor 2.5. Synchronisation Release - Ukraine and Moldova (17th March 2022) =============================================================== diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 9a99d314..ea95fd94 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -324,7 +324,7 @@ def attach_conventional_generators(n, costs, ppl, conventional_carriers, extenda marginal_cost=ppl.marginal_cost, capital_cost=ppl.capital_cost, build_year=ppl.datein.fillna(0).astype(int), - lifetime=(ppl.dateout - ppl.datein).fillna(9999).astype(int), + lifetime=(ppl.dateout - ppl.datein).fillna(np.inf), ) for carrier in conventional_config: @@ -446,7 +446,7 @@ def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **con def attach_extendable_generators(n, costs, ppl, carriers): - logger.warning("The function `attach_extendable_generators` is deprecated in v0.0.5.") + logger.warning("The function `attach_extendable_generators` is deprecated in v0.5.0.") _add_missing_carriers_from_costs(n, costs, carriers) for tech in carriers: From b56d1f6f4d445536708779f3c886ddfec647aabf Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 27 Jun 2022 17:35:19 +0200 Subject: [PATCH 54/61] conventional config section: update to more general attribute assignment scheme --- Snakefile | 5 ++-- config.default.yaml | 2 +- ...{nuclear_eafs.csv => nuclear_p_max_pu.csv} | 0 doc/configuration.rst | 13 ++++++++++- doc/release_notes.rst | 6 ++--- scripts/add_electricity.py | 23 +++++++++++-------- 6 files changed, 32 insertions(+), 17 deletions(-) rename data/{nuclear_eafs.csv => nuclear_p_max_pu.csv} (100%) diff --git a/Snakefile b/Snakefile index af2e6e90..71c6bc21 100644 --- a/Snakefile +++ b/Snakefile @@ -174,7 +174,7 @@ rule build_renewable_profiles: input: base_network="networks/base.nc", corine="data/bundle/corine/g250_clc06_V18_5.tif", - natura=lambda w: ("data/Natura2000_end2020.gpkg" + natura=lambda w: ("resources/natura.tiff" if config["renewable"][w.technology]["natura"] else []), gebco=lambda w: ("data/bundle/GEBCO_2014_2D.nc" @@ -217,7 +217,8 @@ rule add_electricity: load='resources/load.csv', nuts3_shapes='resources/nuts3_shapes.geojson', **{f"profile_{tech}": f"resources/profile_{tech}.nc" - for tech in config['renewable']} + for tech in config['renewable']}, + **{"conventional_{carrier}_{attrs}": fn for carrier in config.get('conventional', {None: {}}).values() for fn in carrier.values() if str(fn).startswith("data/")}, output: "networks/elec.nc" log: "logs/add_electricity.log" benchmark: "benchmarks/add_electricity" diff --git a/config.default.yaml b/config.default.yaml index 177c5e74..a67562c6 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -187,7 +187,7 @@ renewable: conventional: nuclear: - energy_availability_factors: "data/nuclear-eafs.csv" # float of file name + p_max_pu: "data/nuclear_p_max_pu.csv" # float of file name lines: types: diff --git a/data/nuclear_eafs.csv b/data/nuclear_p_max_pu.csv similarity index 100% rename from data/nuclear_eafs.csv rename to data/nuclear_p_max_pu.csv diff --git a/doc/configuration.rst b/doc/configuration.rst index 67d25228..c332ea7d 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -171,7 +171,7 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia .. literalinclude:: ../config.default.yaml :language: yaml :start-at: hydro: - :end-before: lines: + :end-before: conventional: .. csv-table:: :header-rows: 1 @@ -180,6 +180,17 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia .. _lines_cf: +``conventional`` +============= + +Define additional generator attribute for conventional carrier types. If a scalar value is given it is applied to all generators. However if a string starting with "data/" is given, the value is interpreted as a path to a csv file with country specific values. Then, the values are read in and applied to all generators of the given carrier in the given country. Note that the value(s) overwrite the existing values in the corresponding section of the ``generators`` dataframe. + +.. literalinclude:: ../config.default.yaml + :language: yaml + :start-at: conventional: + :end-before: lines: + + ``lines`` ============= diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 57c0a2f0..3addf3ab 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -53,11 +53,11 @@ Upcoming Release * Add function to add global constraint on use of gas in :mod:`prepare_network`. This can be activated by including the keyword ``CH4L`` in the ``{opts}`` wildcard which enforces the limit set in ``electricity: gaslimit:`` given in MWh thermal. Alternatively, it is possible to append a number in the `{opts}` wildcard, e.g. `CH4L200` which limits the gas use to 200 TWh thermal. -* Add configuration option to implement Energy Availability Factors (EAFs) for conventional generation technologies. - * A new section ``conventional`` was added to the config file. This section contains configurations for conventional carriers. -* Implement country-specific EAFs for nuclear power plants based on IAEA 2018-2020 reported country averages. These are specified ``data/nuclear_eafs.csv`` and translate to static ``p_max_pu`` values. +* Add configuration option to implement arbitrary generator attributes for conventional generation technologies. + +* Implement country-specific Energy Availability Factors (EAFs) for nuclear power plants based on IAEA 2018-2020 reported country averages. These are specified ``data/nuclear_p_max_pu.csv`` and translate to static ``p_max_pu`` values. * The powerplants that have been shut down before 2021 are filtered out. diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index ea95fd94..88c2ce66 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -331,17 +331,20 @@ def attach_conventional_generators(n, costs, ppl, conventional_carriers, extenda # Generators with technology affected idx = n.generators.query("carrier == @carrier").index - factors = conventional_config[carrier].get("energy_availability_factors") - if isinstance(factors, float): - # Single value affecting all generators of technology k indiscriminantely of country - n.generators.loc[idx, "p_max_pu"] = factors - elif isinstance(factors, str): - factors = pd.read_csv(factors, index_col=0)['factor'] - # Values affecting generators of technology k country-specific - # First map generator buses to countries; then map countries to p_max_pu - bus_factors = n.buses.country.map(factors) - n.generators.p_max_pu.update(n.generators.loc[idx].bus.map(bus_factors).dropna()) + for key in list(set(conventional_carriers[carrier]) & set(n.generators)): + + values = conventional_config[carrier][key] + + if isinstance(values, str) and str(values).startswith("data/"): + # Values affecting generators of technology k country-specific + # First map generator buses to countries; then map countries to p_max_pu + values = pd.read_csv(values, index_col=0).iloc[:, 0] + bus_values = n.buses.country.map(values) + n.generators[key].update(n.generators.loc[idx].bus.map(bus_values).dropna()) + else: + # Single value affecting all generators of technology k indiscriminantely of country + n.generators.loc[idx, key] = values From 9d997fbd790321dee611cd0dbb49683e6ce1cd53 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 28 Jun 2022 10:14:26 +0200 Subject: [PATCH 55/61] generalize conventional attr handling through config --- Snakefile | 3 ++- config.default.yaml | 2 +- ...{nuclear_eafs.csv => nuclear_p_max_pu.csv} | 0 doc/configuration.rst | 13 ++++++++++- doc/release_notes.rst | 6 ++--- scripts/add_electricity.py | 23 +++++++++++-------- 6 files changed, 31 insertions(+), 16 deletions(-) rename data/{nuclear_eafs.csv => nuclear_p_max_pu.csv} (100%) diff --git a/Snakefile b/Snakefile index af2e6e90..22b6107d 100644 --- a/Snakefile +++ b/Snakefile @@ -217,7 +217,8 @@ rule add_electricity: load='resources/load.csv', nuts3_shapes='resources/nuts3_shapes.geojson', **{f"profile_{tech}": f"resources/profile_{tech}.nc" - for tech in config['renewable']} + for tech in config['renewable']}, + **{"conventional_{carrier}_{attrs}": fn for carrier in config.get('conventional', {None: {}}).values() for fn in carrier.values() if str(fn).startswith("data/")}, output: "networks/elec.nc" log: "logs/add_electricity.log" benchmark: "benchmarks/add_electricity" diff --git a/config.default.yaml b/config.default.yaml index 177c5e74..a67562c6 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -187,7 +187,7 @@ renewable: conventional: nuclear: - energy_availability_factors: "data/nuclear-eafs.csv" # float of file name + p_max_pu: "data/nuclear_p_max_pu.csv" # float of file name lines: types: diff --git a/data/nuclear_eafs.csv b/data/nuclear_p_max_pu.csv similarity index 100% rename from data/nuclear_eafs.csv rename to data/nuclear_p_max_pu.csv diff --git a/doc/configuration.rst b/doc/configuration.rst index 67d25228..c332ea7d 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -171,7 +171,7 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia .. literalinclude:: ../config.default.yaml :language: yaml :start-at: hydro: - :end-before: lines: + :end-before: conventional: .. csv-table:: :header-rows: 1 @@ -180,6 +180,17 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia .. _lines_cf: +``conventional`` +============= + +Define additional generator attribute for conventional carrier types. If a scalar value is given it is applied to all generators. However if a string starting with "data/" is given, the value is interpreted as a path to a csv file with country specific values. Then, the values are read in and applied to all generators of the given carrier in the given country. Note that the value(s) overwrite the existing values in the corresponding section of the ``generators`` dataframe. + +.. literalinclude:: ../config.default.yaml + :language: yaml + :start-at: conventional: + :end-before: lines: + + ``lines`` ============= diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 57c0a2f0..3addf3ab 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -53,11 +53,11 @@ Upcoming Release * Add function to add global constraint on use of gas in :mod:`prepare_network`. This can be activated by including the keyword ``CH4L`` in the ``{opts}`` wildcard which enforces the limit set in ``electricity: gaslimit:`` given in MWh thermal. Alternatively, it is possible to append a number in the `{opts}` wildcard, e.g. `CH4L200` which limits the gas use to 200 TWh thermal. -* Add configuration option to implement Energy Availability Factors (EAFs) for conventional generation technologies. - * A new section ``conventional`` was added to the config file. This section contains configurations for conventional carriers. -* Implement country-specific EAFs for nuclear power plants based on IAEA 2018-2020 reported country averages. These are specified ``data/nuclear_eafs.csv`` and translate to static ``p_max_pu`` values. +* Add configuration option to implement arbitrary generator attributes for conventional generation technologies. + +* Implement country-specific Energy Availability Factors (EAFs) for nuclear power plants based on IAEA 2018-2020 reported country averages. These are specified ``data/nuclear_p_max_pu.csv`` and translate to static ``p_max_pu`` values. * The powerplants that have been shut down before 2021 are filtered out. diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index ea95fd94..88c2ce66 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -331,17 +331,20 @@ def attach_conventional_generators(n, costs, ppl, conventional_carriers, extenda # Generators with technology affected idx = n.generators.query("carrier == @carrier").index - factors = conventional_config[carrier].get("energy_availability_factors") - if isinstance(factors, float): - # Single value affecting all generators of technology k indiscriminantely of country - n.generators.loc[idx, "p_max_pu"] = factors - elif isinstance(factors, str): - factors = pd.read_csv(factors, index_col=0)['factor'] - # Values affecting generators of technology k country-specific - # First map generator buses to countries; then map countries to p_max_pu - bus_factors = n.buses.country.map(factors) - n.generators.p_max_pu.update(n.generators.loc[idx].bus.map(bus_factors).dropna()) + for key in list(set(conventional_carriers[carrier]) & set(n.generators)): + + values = conventional_config[carrier][key] + + if isinstance(values, str) and str(values).startswith("data/"): + # Values affecting generators of technology k country-specific + # First map generator buses to countries; then map countries to p_max_pu + values = pd.read_csv(values, index_col=0).iloc[:, 0] + bus_values = n.buses.country.map(values) + n.generators[key].update(n.generators.loc[idx].bus.map(bus_values).dropna()) + else: + # Single value affecting all generators of technology k indiscriminantely of country + n.generators.loc[idx, key] = values From 917f41ef21d97635a1ff1aa8ebca2d7ca0a1255d Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Tue, 28 Jun 2022 13:11:08 +0200 Subject: [PATCH 56/61] Update Snakefile Co-authored-by: Fabian Neumann --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index 22b6107d..c786efdc 100644 --- a/Snakefile +++ b/Snakefile @@ -218,7 +218,7 @@ rule add_electricity: nuts3_shapes='resources/nuts3_shapes.geojson', **{f"profile_{tech}": f"resources/profile_{tech}.nc" for tech in config['renewable']}, - **{"conventional_{carrier}_{attrs}": fn for carrier in config.get('conventional', {None: {}}).values() for fn in carrier.values() if str(fn).startswith("data/")}, + **{f"conventional_{carrier}_{attrs}": fn for carrier in config.get('conventional', {None: {}}).values() for attrs, fn in carrier.items() if str(fn).startswith("data/")}, output: "networks/elec.nc" log: "logs/add_electricity.log" benchmark: "benchmarks/add_electricity" From 3294ad92ebafaa5ace5b3fcbdf2f3d29efe2537f Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Tue, 28 Jun 2022 13:14:47 +0200 Subject: [PATCH 57/61] Update scripts/solve_network.py Co-authored-by: Martha Frysztacki --- scripts/solve_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 6c40ca4f..b3280a94 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -104,7 +104,7 @@ def prepare_network(n, solve_opts): if load_shedding: n.add("Carrier", "load", color="#dd2e23", nice_name="Load shedding") buses_i = n.buses.query("carrier == 'AC'").index - if not np.isscalar(load_shedding): load_shedding = 1e2 + if not np.isscalar(load_shedding): load_shedding = 1e2 # Eur/kWh # intersect between macroeconomic and surveybased # willingness to pay # http://journal.frontiersin.org/article/10.3389/fenrg.2015.00055/full) From 522f218eed4cacd8d2d37ea3b034ca6b38e60c52 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 28 Jun 2022 13:15:37 +0200 Subject: [PATCH 58/61] Snakefile: rename attrs to attr in add_electricity input function --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index c786efdc..9e8d6236 100644 --- a/Snakefile +++ b/Snakefile @@ -218,7 +218,7 @@ rule add_electricity: nuts3_shapes='resources/nuts3_shapes.geojson', **{f"profile_{tech}": f"resources/profile_{tech}.nc" for tech in config['renewable']}, - **{f"conventional_{carrier}_{attrs}": fn for carrier in config.get('conventional', {None: {}}).values() for attrs, fn in carrier.items() if str(fn).startswith("data/")}, + **{f"conventional_{carrier}_{attr}": fn for carrier in config.get('conventional', {None: {}}).values() for attr, fn in carrier.items() if str(fn).startswith("data/")}, output: "networks/elec.nc" log: "logs/add_electricity.log" benchmark: "benchmarks/add_electricity" From 82013fd0816c380e148e4b214408254d84f8c946 Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Tue, 28 Jun 2022 13:16:55 +0200 Subject: [PATCH 59/61] Update scripts/add_electricity.py Co-authored-by: Martha Frysztacki --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 88c2ce66..d244a381 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -332,7 +332,7 @@ def attach_conventional_generators(n, costs, ppl, conventional_carriers, extenda # Generators with technology affected idx = n.generators.query("carrier == @carrier").index - for key in list(set(conventional_carriers[carrier]) & set(n.generators)): + for key in list(set(conventional_config[carrier]) & set(n.generators)): values = conventional_config[carrier][key] From e9e00291c501b473146d12098510be3e4e96eccc Mon Sep 17 00:00:00 2001 From: martacki Date: Tue, 28 Jun 2022 13:40:14 +0200 Subject: [PATCH 60/61] update configtables/opts to include CH4L option --- doc/configtables/opts.csv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/configtables/opts.csv b/doc/configtables/opts.csv index 918d0d17..6a72dd01 100644 --- a/doc/configtables/opts.csv +++ b/doc/configtables/opts.csv @@ -8,4 +8,5 @@ Trigger, Description, Definition, Status ``ATK``, "Require each node to be autarkic. Example: ``ATK`` removes all lines and links. ``ATKc`` removes all cross-border lines and links.", ``prepare_network``, In active use ``BAU``, Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities``, ``solve_network``: `add_opts_constraints() `__, Untested ``SAFE``, Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network., ``solve_network`` `add_opts_constraints() `__, Untested -``carrier+{c|p}factor``, "Alter the capital cost (``c``) or installable potential (``p``) of a carrier by a factor. Example: ``solar+c0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use +``carrier+{c|p|m}factor``,"Alter the capital cost (``c``), installable potential (``p``) or marginal costs (``m``) of a carrier by a factor. Example: ``solar+c0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use +``CH4L``,"Add an overall absolute gas limit. If configured in ``electricity: gaslimit`` it is given in MWh thermal, if a float is appended, the overall gaslimit is assumed to be given in TWh thermal (e.g. ``CH4L200`` limits gas dispatch to 200 TWh termal)", ``prepare_network``: ``add_gaslimit()``, In active use \ No newline at end of file From 67ac464b6a0b66e012b1dcaae9748c0c5722e0a6 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 28 Jun 2022 16:33:46 +0200 Subject: [PATCH 61/61] add_electricity: use conventional_inputs from snakemake.input for attach_conventional_generators --- Snakefile | 2 +- scripts/add_electricity.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Snakefile b/Snakefile index a8d98057..576d1fbb 100644 --- a/Snakefile +++ b/Snakefile @@ -229,7 +229,7 @@ rule add_electricity: nuts3_shapes='resources/nuts3_shapes.geojson', **{f"profile_{tech}": f"resources/profile_{tech}.nc" for tech in config['renewable']}, - **{f"conventional_{carrier}_{attr}": fn for carrier in config.get('conventional', {None: {}}).values() for attr, fn in carrier.items() if str(fn).startswith("data/")}, + **{f"conventional_{carrier}_{attr}": fn for carrier, d in config.get('conventional', {None: {}}).items() for attr, fn in d.items() if str(fn).startswith("data/")}, output: "networks/elec.nc" log: "logs/add_electricity.log" benchmark: "benchmarks/add_electricity" diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index d244a381..342b12e9 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -302,7 +302,7 @@ def attach_wind_and_solar(n, costs, input_profiles, technologies, extendable_car p_max_pu=ds['profile'].transpose('time', 'bus').to_pandas()) -def attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers, conventional_config): +def attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers, conventional_config, conventional_inputs): carriers = set(conventional_carriers) | set(extendable_carriers['Generator']) _add_missing_carriers_from_costs(n, costs, carriers) @@ -332,19 +332,19 @@ def attach_conventional_generators(n, costs, ppl, conventional_carriers, extenda # Generators with technology affected idx = n.generators.query("carrier == @carrier").index - for key in list(set(conventional_config[carrier]) & set(n.generators)): + for attr in list(set(conventional_config[carrier]) & set(n.generators)): - values = conventional_config[carrier][key] + values = conventional_config[carrier][attr] - if isinstance(values, str) and str(values).startswith("data/"): + if f"conventional_{carrier}_{attr}" in conventional_inputs: # Values affecting generators of technology k country-specific # First map generator buses to countries; then map countries to p_max_pu values = pd.read_csv(values, index_col=0).iloc[:, 0] bus_values = n.buses.country.map(values) - n.generators[key].update(n.generators.loc[idx].bus.map(bus_values).dropna()) + n.generators[attr].update(n.generators.loc[idx].bus.map(bus_values).dropna()) else: # Single value affecting all generators of technology k indiscriminantely of country - n.generators.loc[idx, key] = values + n.generators.loc[idx, attr] = values @@ -603,7 +603,8 @@ if __name__ == "__main__": update_transmission_costs(n, costs, snakemake.config['lines']['length_factor']) - attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers, snakemake.config.get("conventional", {})) + conventional_inputs = {k: v for k, v in snakemake.input.items() if k.startswith("conventional_")} + attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers, snakemake.config.get("conventional", {}), conventional_inputs) attach_wind_and_solar(n, costs, snakemake.input, renewable_carriers, extendable_carriers, snakemake.config['lines']['length_factor'])