From 299e71e2b3eacf53f796db3c9e965034bbb25d16 Mon Sep 17 00:00:00 2001 From: martacki Date: Wed, 17 Nov 2021 13:46:33 +0100 Subject: [PATCH 01/90] introduce hierarchical agglomeratice clustering (hac) --- config.default.yaml | 9 +++-- config.tutorial.yaml | 9 +++-- doc/configtables/clustering.csv | 9 +++-- envs/environment.yaml | 2 +- scripts/cluster_network.py | 63 +++++++++++++++++++++++++++++---- scripts/simplify_network.py | 10 ++++-- test/config.test1.yaml | 9 +++-- 7 files changed, 93 insertions(+), 18 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index f70e7c2c..631645b9 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -20,8 +20,13 @@ scenario: countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'] clustering: - simplify: - to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + simplify_network: + to_substations: true # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + algorithm: hac + feature: solar+onwind-time + cluster_network: + algorithm: hac # choose from: [hac, kmeans] + feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc. snapshots: start: "2013-01-01" diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 26ead242..919da193 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -20,8 +20,13 @@ scenario: countries: ['DE'] clustering: - simplify: - to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + simplify_network: + to_substations: true # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + algorithm: hac + feature: solar+onwind-time + cluster_network: + algorithm: hac # choose from: [hac, kmeans] + feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc. snapshots: start: "2013-03-01" diff --git a/doc/configtables/clustering.csv b/doc/configtables/clustering.csv index 2f63f955..e488dd39 100644 --- a/doc/configtables/clustering.csv +++ b/doc/configtables/clustering.csv @@ -1,3 +1,8 @@ ,Unit,Values,Description -simplify,,, --- to_substations,bool,"{'true','false'}","Aggregates all nodes without power injection (positive or negative, i.e. demand or generation) to electrically closest ones" +simplify_network,,, +-- to_substations,bool,"One of {'true','false'}","Aggregates all nodes without power injection (positive or negative, i.e. demand or generation) to electrically closest ones" +-- algorithm,str,"One of {‘kmenas’, ‘hac’}", +-- feature,str,"Str in the format ‘carrier1+carrier2+...+carrierN-X’, where CarrierI can be from {‘solar’, ‘onwind’, ‘offwind’, ‘ror’} and X is one of {‘cap’, ‘time’}.", +cluster_network,,, +-- algorithm,str,"One of {‘kmenas’, ‘hac’}", +-- feature,str,"Str in the format ‘carrier1+carrier2+...+carrierN-X’, where CarrierI can be from {‘solar’, ‘onwind’, ‘offwind’, ‘ror’} and X is one of {‘cap’, ‘time’}.", diff --git a/envs/environment.yaml b/envs/environment.yaml index 29d743ac..5f6a8310 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -12,7 +12,6 @@ dependencies: - pip - mamba # esp for windows build - - pypsa>=0.18 - atlite>=0.2.5 - dask<=2021.3.1 # until https://github.com/dask/dask/issues/7583 is solved @@ -56,5 +55,6 @@ dependencies: - tabula-py - pip: + - git+https://github.com/pypsa/pypsa.git#egg=pypsa - vresutils==0.3.1 - tsam>=1.1.0 diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 980b73b0..7aae3ae6 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -138,7 +138,7 @@ import seaborn as sns from functools import reduce from pypsa.networkclustering import (busmap_by_kmeans, busmap_by_spectral_clustering, - _make_consense, get_clustering_from_busmap) + busmap_by_hac, _make_consense, get_clustering_from_busmap) from add_electricity import load_costs @@ -170,6 +170,45 @@ def weighting_for_country(n, x): return (w * (100. / w.max())).clip(lower=1.).astype(int) +def get_feature_for_hac(n, buses_i, feature=None): #buses_i = n.buses.index + + if feature is None: + feature = "solar+onwind-time" + + carriers = feature.split('-')[0].split('+') + if "offwind" in carriers: + carriers.remove("offwind") + carriers = np.append(carriers, network.generators.carrier.filter(like='offwind').unique()) + + if feature.split('-')[1] == 'cap': + feature_data = pd.DataFrame(index=buses_i, columns=carriers) + for carrier in carriers: + try: + feature_data[carrier] = (n.generators_t.p_max_pu.filter(like=carrier).mean() + .rename(index=lambda x: x.split(' ')[0])) + except: + feature_data[carrier] = (n.generators_t.p_max_pu.filter(like=carrier).mean() + .rename(index=lambda x: x.split(' ')[0] + ' ' + x.split(' ')[1])) + + if feature.split('-')[1] == 'time': + feature_data = pd.DataFrame(columns=buses_i) + for carrier in carriers: + try: + # without simpl wildcard (bus names are "X X"): + feature_data = feature_data.append(n.generators_t.p_max_pu.filter(like=carrier) + .rename(columns=lambda x: x.split(' ')[0]))[buses_i] + except: + # with simpl wildcard (bus names are "X X X"): + feature_data = feature_data.append(n.generators_t.p_max_pu.filter(like=carrier) + .rename(columns=lambda x: x.split(' ')[0] + ' ' + x.split(' ')[1]))[buses_i] + feature_data = feature_data.T + feature_data.columns = feature_data.columns.astype(str) # Timestamp will raise error in sklearn>=v1.2 + + feature_data = feature_data.fillna(0) + + return feature_data + + def distribute_clusters(n, n_clusters, focus_weights=None, solver_name=None): """Determine the number of clusters per country""" @@ -221,12 +260,18 @@ def distribute_clusters(n, n_clusters, focus_weights=None, solver_name=None): return pd.Series(m.n.get_values(), index=L.index).astype(int) -def busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights=None, algorithm="kmeans", **algorithm_kwds): +def busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights=None, algorithm="kmeans", feature=None, **algorithm_kwds): if algorithm == "kmeans": algorithm_kwds.setdefault('n_init', 1000) algorithm_kwds.setdefault('max_iter', 30000) algorithm_kwds.setdefault('tol', 1e-6) + if algorithm == "hac": + feature = get_feature_for_hac(n, buses_i=n.buses.index, feature=feature) + elif feature is not None: + logger.info(f"keyword argument feature is only valid for algorithm 'hac'." + f"given feature {feature} will be ignored.") + n.determine_network_topology() n_clusters = distribute_clusters(n, n_clusters, focus_weights=focus_weights, solver_name=solver_name) @@ -250,8 +295,10 @@ def busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights=None, algori return prefix + busmap_by_spectral_clustering(reduce_network(n, x), n_clusters[x.name], **algorithm_kwds) elif algorithm == "louvain": return prefix + busmap_by_louvain(reduce_network(n, x), n_clusters[x.name], **algorithm_kwds) + elif algorithm == "hac": + return prefix + busmap_by_hac(n, n_clusters[x.name], buses_i=x.index, feature=feature.loc[x.index]) else: - raise ValueError(f"`algorithm` must be one of 'kmeans', 'spectral' or 'louvain'. Is {algorithm}.") + raise ValueError(f"`algorithm` must be one of 'kmeans', 'hac', 'spectral' or 'louvain'. Is {algorithm}.") return (n.buses.groupby(['country', 'sub_network'], group_keys=False) .apply(busmap_for_country).squeeze().rename('busmap')) @@ -259,7 +306,9 @@ def busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights=None, algori def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carriers=None, line_length_factor=1.25, potential_mode='simple', solver_name="cbc", - algorithm="kmeans", extended_link_costs=0, focus_weights=None): + algorithm="kmeans", feature=None, extended_link_costs=0, focus_weights=None): + + logger.info(f"Clustering network using algorithm {algorithm} and feature {feature}...") if potential_mode == 'simple': p_nom_max_strategy = np.sum @@ -273,7 +322,7 @@ def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carr busmap.index = busmap.index.astype(str) logger.info(f"Imported custom busmap from {snakemake.input.custom_busmap}") else: - busmap = busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights, algorithm) + busmap = busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights, algorithm, feature) clustering = get_clustering_from_busmap( n, busmap, @@ -313,7 +362,7 @@ def cluster_regions(busmaps, input=None, output=None): for which in ('regions_onshore', 'regions_offshore'): regions = gpd.read_file(getattr(input, which)).set_index('name') - geom_c = regions.geometry.groupby(busmap).apply(shapely.ops.cascaded_union) + geom_c = regions.geometry.groupby(busmap).apply(shapely.ops.unary_union) regions_c = gpd.GeoDataFrame(dict(geometry=geom_c)) regions_c.index.name = 'name' save_to_geojson(regions_c, getattr(output, which)) @@ -377,6 +426,8 @@ if __name__ == "__main__": line_length_factor=line_length_factor, potential_mode=potential_mode, solver_name=snakemake.config['solving']['solver']['name'], + algorithm=snakemake.config.get('clustering', {}).get('cluster_network', {}).get('algorithm', 'kmeans'), + feature=snakemake.config.get('clustering', {}).get('cluster_network', {}).get('feature', None), extended_link_costs=hvac_overhead_cost, focus_weights=focus_weights) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 85bc4d15..8a93952c 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -360,7 +360,7 @@ def aggregate_to_substations(n, buses_i=None): return clustering.network, busmap -def cluster(n, n_clusters): +def cluster(n, n_clusters, algorithm="kmeans", feature=None): logger.info(f"Clustering to {n_clusters} buses") focus_weights = snakemake.config.get('focus_weights', None) @@ -377,8 +377,10 @@ def cluster(n, n_clusters): potential_mode = (consense(pd.Series([snakemake.config['renewable'][tech]['potential'] for tech in renewable_carriers])) if len(renewable_carriers) > 0 else 'conservative') + clustering = clustering_for_n_clusters(n, n_clusters, custom_busmap=False, potential_mode=potential_mode, solver_name=snakemake.config['solving']['solver']['name'], + algorithm=algorithm, feature=feature, focus_weights=focus_weights) return clustering.network, clustering.busmap @@ -400,12 +402,14 @@ if __name__ == "__main__": busmaps = [trafo_map, simplify_links_map, stub_map] - if snakemake.config.get('clustering', {}).get('simplify', {}).get('to_substations', False): + if snakemake.config.get('clustering', {}).get('simplify_network', {}).get('to_substations', False): n, substation_map = aggregate_to_substations(n) busmaps.append(substation_map) if snakemake.wildcards.simpl: - n, cluster_map = cluster(n, int(snakemake.wildcards.simpl)) + n, cluster_map = cluster(n, int(snakemake.wildcards.simpl), + algorithm=snakemake.config.get('clustering', {}).get('simplify_network', {}).get('algorithm', 'hac'), + feature=snakemake.config.get('clustering', {}).get('simplify_network', {}).get('feature', None)) busmaps.append(cluster_map) # some entries in n.buses are not updated in previous functions, therefore can be wrong. as they are not needed diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 2986037b..0c34ea13 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -19,8 +19,13 @@ scenario: countries: ['DE'] clustering: - simplify: - to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + simplify_network: + to_substations: true # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + algorithm: hac + feature: solar+onwind-time + cluster_network: + algorithm: hac # choose from: [hac, kmeans] + feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc. snapshots: start: "2013-03-01" From 6c4ea69e9563c257961e10f3e8d37b85d0cff7cf Mon Sep 17 00:00:00 2001 From: martacki Date: Fri, 10 Dec 2021 17:53:40 +0100 Subject: [PATCH 02/90] clustering: own config for clustering settings --- scripts/cluster_network.py | 14 +++++++++----- scripts/simplify_network.py | 5 +++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 7aae3ae6..415d3820 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -170,7 +170,10 @@ def weighting_for_country(n, x): return (w * (100. / w.max())).clip(lower=1.).astype(int) -def get_feature_for_hac(n, buses_i, feature=None): #buses_i = n.buses.index +def get_feature_for_hac(n, buses_i=None, feature=None): + + if buses_i is None: + buses_i = n.buses.index if feature is None: feature = "solar+onwind-time" @@ -269,8 +272,8 @@ def busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights=None, algori if algorithm == "hac": feature = get_feature_for_hac(n, buses_i=n.buses.index, feature=feature) elif feature is not None: - logger.info(f"keyword argument feature is only valid for algorithm 'hac'." - f"given feature {feature} will be ignored.") + logger.warning(f"Keyword argument feature is only valid for algorithm 'hac'." + f"given feature {feature} will be ignored.") n.determine_network_topology() @@ -422,12 +425,13 @@ if __name__ == "__main__": potential_mode = consense(pd.Series([snakemake.config['renewable'][tech]['potential'] for tech in renewable_carriers])) custom_busmap = snakemake.config["enable"].get("custom_busmap", False) + cluster_config = snakemake.config.get('clustering', {}).get('cluster_network', {}) clustering = clustering_for_n_clusters(n, n_clusters, custom_busmap, aggregate_carriers, line_length_factor=line_length_factor, potential_mode=potential_mode, solver_name=snakemake.config['solving']['solver']['name'], - algorithm=snakemake.config.get('clustering', {}).get('cluster_network', {}).get('algorithm', 'kmeans'), - feature=snakemake.config.get('clustering', {}).get('cluster_network', {}).get('feature', None), + algorithm=cluster_config.get('algorithm', 'kmeans'), + feature=cluster_config.get('feature', None), extended_link_costs=hvac_overhead_cost, focus_weights=focus_weights) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 8a93952c..9464c3f7 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -407,9 +407,10 @@ if __name__ == "__main__": busmaps.append(substation_map) if snakemake.wildcards.simpl: + cluster_config = snakemake.config.get('clustering', {}).get('simplify_network', {}) n, cluster_map = cluster(n, int(snakemake.wildcards.simpl), - algorithm=snakemake.config.get('clustering', {}).get('simplify_network', {}).get('algorithm', 'hac'), - feature=snakemake.config.get('clustering', {}).get('simplify_network', {}).get('feature', None)) + algorithm=cluster_config.get('algorithm', 'hac'), + feature=cluster_config.get('feature', None)) busmaps.append(cluster_map) # some entries in n.buses are not updated in previous functions, therefore can be wrong. as they are not needed From 256ac48b470470571d0b5751319e8f441ce39e0c Mon Sep 17 00:00:00 2001 From: martacki Date: Fri, 4 Feb 2022 16:45:00 +0100 Subject: [PATCH 03/90] resolve merging master bugs --- scripts/cluster_network.py | 13 +++++++++---- scripts/simplify_network.py | 6 +++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index d4b3b139..1c55234c 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -190,9 +190,11 @@ def get_feature_for_hac(n, buses_i=None, feature=None): feature_data = pd.DataFrame(index=buses_i, columns=carriers) for carrier in carriers: try: + # without simpl wildcard (bus names are "X X"): feature_data[carrier] = (n.generators_t.p_max_pu.filter(like=carrier).mean() .rename(index=lambda x: x.split(' ')[0])) except: + # with simpl wildcard (bus names are "X X X"): feature_data[carrier] = (n.generators_t.p_max_pu.filter(like=carrier).mean() .rename(index=lambda x: x.split(' ')[0] + ' ' + x.split(' ')[1])) @@ -208,7 +210,7 @@ def get_feature_for_hac(n, buses_i=None, feature=None): feature_data = feature_data.append(n.generators_t.p_max_pu.filter(like=carrier) .rename(columns=lambda x: x.split(' ')[0] + ' ' + x.split(' ')[1]))[buses_i] feature_data = feature_data.T - feature_data.columns = feature_data.columns.astype(str) # Timestamp will raise error in sklearn>=v1.2 + feature_data.columns = feature_data.columns.astype(str) # timestamp raises error in sklearn>=v1.2 feature_data = feature_data.fillna(0) @@ -309,9 +311,9 @@ def busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights=None, algori def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carriers=None, line_length_factor=1.25, potential_mode='simple', solver_name="cbc", - algorithm="kmeans", feature=None, extended_link_costs=0, focus_weights=None): + algorithm="hac", feature=None, extended_link_costs=0, focus_weights=None): - logger.info(f"Clustering network using algorithm {algorithm} and feature {feature}...") + logger.info(f"Clustering network using algorithm ``{algorithm}`` and feature ``{feature}``...") if potential_mode == 'simple': p_nom_max_strategy = np.sum @@ -424,10 +426,13 @@ if __name__ == "__main__": custom_busmap.index = custom_busmap.index.astype(str) logger.info(f"Imported custom busmap from {snakemake.input.custom_busmap}") + cluster_config = snakemake.config.get('clustering', {}).get('cluster_network', {}) clustering = clustering_for_n_clusters(n, n_clusters, custom_busmap, aggregate_carriers, line_length_factor, potential_mode, snakemake.config['solving']['solver']['name'], - "kmeans", hvac_overhead_cost, focus_weights) + cluster_config.get("algorithm", "hac"), + cluster_config.get("feature", "solar+onwind-time"), + hvac_overhead_cost, focus_weights) update_p_nom_max(n) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 0e9dd385..23facc79 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -353,11 +353,10 @@ def aggregate_to_substations(n, buses_i=None): line_length_factor=1.0, generator_strategies={'p_nom_max': 'sum'}, scale_link_capital_costs=False) - return clustering.network, busmap -def cluster(n, n_clusters, config, algorithm="kmeans", feature=None): +def cluster(n, n_clusters, config, algorithm="hac", feature=None): logger.info(f"Clustering to {n_clusters} buses") focus_weights = config.get('focus_weights', None) @@ -403,7 +402,8 @@ if __name__ == "__main__": busmaps = [trafo_map, simplify_links_map, stub_map] - if snakemake.config.get('clustering', {}).get('simplify_network', {}).get('to_substations', False): + cluster_config = snakemake.config.get('clustering', {}).get('simplify_network', {}) + if cluster_config.get('to_substations', False): n, substation_map = aggregate_to_substations(n) busmaps.append(substation_map) From b5dbf4eb3240222e3a3145bbf87b3d66a63d7169 Mon Sep 17 00:00:00 2001 From: martacki Date: Fri, 4 Feb 2022 17:19:23 +0100 Subject: [PATCH 04/90] overwrite country of isolated buses --- scripts/cluster_network.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 1c55234c..bd49556f 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -271,11 +271,36 @@ def busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights=None, algori algorithm_kwds.setdefault('max_iter', 30000) algorithm_kwds.setdefault('tol', 1e-6) + def fix_country_assignment_for_hac(n): + # overwrite country of nodes that are disconnected from their country-topology + for country in n.buses.country.unique(): + m = n.copy() + m.buses = m.buses.query("country in @country") + m.lines = m.lines.query("bus0 in @m.buses.index and bus1 in @m.buses.index") + m.links = m.links.query("bus0 in @m.buses.index and bus1 in @m.buses.index") + + _, labels = csgraph.connected_components(m.adjacency_matrix(), directed=False) + component = pd.Series(labels, index=m.buses.index) + component_sizes = component.value_counts() + + if len(component_sizes)>1: + disconnected_bus = component[component==component_sizes[component_sizes==component_sizes.min()].index[0]].index + neighbor_bus = n.lines.query("bus0 in @disconnected_bus or bus1 in @disconnected_bus").iloc[0][['bus0','bus1']] + new_country = list(set(n.buses.loc[neighbor_bus].country)-set([country]))[0] + + logger.info(f"overwriting country ``{country}`` of bus ``{disconnected_bus}`` to new country ``{new_country}``, " + "because it is disconnected from its inital inter-country transmission grid.") + n.buses.at[disconnected_bus, "country"] = new_country + return n + if algorithm == "hac": + from scipy.sparse import csgraph + feature = get_feature_for_hac(n, buses_i=n.buses.index, feature=feature) + n = fix_country_assignment_for_hac(n) elif feature is not None: logger.warning(f"Keyword argument feature is only valid for algorithm 'hac'." - f"given feature {feature} will be ignored.") + f"given feature ``{feature}`` will be ignored.") n.determine_network_topology() From 82a0338e9f3f8b68eb7d73aad0ceb3b26862d028 Mon Sep 17 00:00:00 2001 From: martacki Date: Fri, 4 Feb 2022 20:27:18 +0100 Subject: [PATCH 05/90] treatment of outliers and small feature-bugfix --- scripts/cluster_network.py | 17 +++++++++-------- scripts/simplify_network.py | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index bd49556f..2cc406eb 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -272,6 +272,8 @@ def busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights=None, algori algorithm_kwds.setdefault('tol', 1e-6) def fix_country_assignment_for_hac(n): + from scipy.sparse import csgraph + # overwrite country of nodes that are disconnected from their country-topology for country in n.buses.country.unique(): m = n.copy() @@ -288,19 +290,18 @@ def busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights=None, algori neighbor_bus = n.lines.query("bus0 in @disconnected_bus or bus1 in @disconnected_bus").iloc[0][['bus0','bus1']] new_country = list(set(n.buses.loc[neighbor_bus].country)-set([country]))[0] - logger.info(f"overwriting country ``{country}`` of bus ``{disconnected_bus}`` to new country ``{new_country}``, " + logger.info(f"overwriting country `{country}` of bus `{disconnected_bus}` to new country `{new_country}`, " "because it is disconnected from its inital inter-country transmission grid.") n.buses.at[disconnected_bus, "country"] = new_country return n if algorithm == "hac": - from scipy.sparse import csgraph - feature = get_feature_for_hac(n, buses_i=n.buses.index, feature=feature) n = fix_country_assignment_for_hac(n) - elif feature is not None: - logger.warning(f"Keyword argument feature is only valid for algorithm 'hac'." - f"given feature ``{feature}`` will be ignored.") + + if (algorithm != "hac") and (feature is not None): + logger.warning(f"Keyword argument feature is only valid for algorithm `hac`. " + f"Given feature `{feature}` will be ignored.") n.determine_network_topology() @@ -338,7 +339,7 @@ def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carr line_length_factor=1.25, potential_mode='simple', solver_name="cbc", algorithm="hac", feature=None, extended_link_costs=0, focus_weights=None): - logger.info(f"Clustering network using algorithm ``{algorithm}`` and feature ``{feature}``...") + logger.info(f"Clustering network using algorithm `{algorithm}` and feature `{feature}`...") if potential_mode == 'simple': p_nom_max_strategy = np.sum @@ -348,7 +349,7 @@ def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carr raise AttributeError(f"potential_mode should be one of 'simple' or 'conservative' but is '{potential_mode}'") if not custom_busmap: - busmap = busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights, algorithm) + busmap = busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights, algorithm, feature) else: busmap = custom_busmap diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 23facc79..b28bc4dc 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -407,7 +407,21 @@ if __name__ == "__main__": n, substation_map = aggregate_to_substations(n) busmaps.append(substation_map) + # treatment of outliers (nodes without a profile for considered carrier) for "cluster_network" + if snakemake.config.get("clustering", {}).get("cluster_network", {}).get("algorithm", "hac") == "hac": + carriers = cluster_config.get("feature", "solar+onwind-time").split('-')[0].split('+') + buses_i = list(set(n.buses.index)-set(n.generators.query("carrier in @carriers").bus)) + n, busmap_hac = aggregate_to_substations(n, buses_i) + busmaps.append(busmap_hac) + if snakemake.wildcards.simpl: + # treatment of outliers (nodes without a profile for a considered carrier) for "simplify" + if cluster_config.get("algorithm", "hac") == "hac": + carriers = cluster_config.get("feature", "solar+onwind-time").split('-')[0].split('+') + buses_i = list(set(n.buses.index)-set(n.generators.query("carrier in @carriers").bus)) + n, busmap_hac = aggregate_to_substations(n, buses_i) + busmaps.append(busmap_hac) + # conduct clustering n, cluster_map = cluster(n, int(snakemake.wildcards.simpl), snakemake.config, algorithm=cluster_config.get('algorithm', 'hac'), feature=cluster_config.get('feature', None)) From f02d5fe82161f7373423c8ff6571a4319ab8c164 Mon Sep 17 00:00:00 2001 From: martacki Date: Mon, 7 Mar 2022 11:43:04 +0100 Subject: [PATCH 06/90] fix clustering setup for hac according to fneum suggestions --- doc/configtables/clustering.csv | 4 ++-- envs/environment.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/configtables/clustering.csv b/doc/configtables/clustering.csv index e488dd39..d14ab395 100644 --- a/doc/configtables/clustering.csv +++ b/doc/configtables/clustering.csv @@ -1,8 +1,8 @@ ,Unit,Values,Description simplify_network,,, -- to_substations,bool,"One of {'true','false'}","Aggregates all nodes without power injection (positive or negative, i.e. demand or generation) to electrically closest ones" --- algorithm,str,"One of {‘kmenas’, ‘hac’}", +-- algorithm,str,"One of {‘kmeans’, ‘hac’}", -- feature,str,"Str in the format ‘carrier1+carrier2+...+carrierN-X’, where CarrierI can be from {‘solar’, ‘onwind’, ‘offwind’, ‘ror’} and X is one of {‘cap’, ‘time’}.", cluster_network,,, --- algorithm,str,"One of {‘kmenas’, ‘hac’}", +-- algorithm,str,"One of {‘kmeans’, ‘hac’}", -- feature,str,"Str in the format ‘carrier1+carrier2+...+carrierN-X’, where CarrierI can be from {‘solar’, ‘onwind’, ‘offwind’, ‘ror’} and X is one of {‘cap’, ‘time’}.", diff --git a/envs/environment.yaml b/envs/environment.yaml index 1d8d9d28..c64b5fbb 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -10,6 +10,7 @@ dependencies: - python>=3.8 - pip + - pypsa>=0.19.1 - atlite>=0.2.5 - dask @@ -53,6 +54,5 @@ dependencies: - tabula-py - pip: - - git+https://github.com/pypsa/pypsa.git#egg=pypsa - vresutils>=0.3.1 - tsam>=1.1.0 From 68f332f5cc8603ceb890fb41f01cfc02bbebb10e Mon Sep 17 00:00:00 2001 From: martacki Date: Thu, 17 Mar 2022 15:42:04 +0100 Subject: [PATCH 07/90] suggestions by coroa and fneum --- scripts/cluster_network.py | 42 +++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index f4058e79..003442ff 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -189,28 +189,20 @@ def get_feature_for_hac(n, buses_i=None, feature=None): if feature.split('-')[1] == 'cap': feature_data = pd.DataFrame(index=buses_i, columns=carriers) for carrier in carriers: - try: - # without simpl wildcard (bus names are "X X"): - feature_data[carrier] = (n.generators_t.p_max_pu.filter(like=carrier).mean() - .rename(index=lambda x: x.split(' ')[0])) - except: - # with simpl wildcard (bus names are "X X X"): - feature_data[carrier] = (n.generators_t.p_max_pu.filter(like=carrier).mean() - .rename(index=lambda x: x.split(' ')[0] + ' ' + x.split(' ')[1])) + gen_i = n.generators.query("carrier == @carrier").index + attach = n.generators_t.p_max_pu[gen_i].mean().rename(index = n.generators.loc[gen_i].bus) + feature_data[carrier] = attach if feature.split('-')[1] == 'time': feature_data = pd.DataFrame(columns=buses_i) for carrier in carriers: - try: - # without simpl wildcard (bus names are "X X"): - feature_data = feature_data.append(n.generators_t.p_max_pu.filter(like=carrier) - .rename(columns=lambda x: x.split(' ')[0]))[buses_i] - except: - # with simpl wildcard (bus names are "X X X"): - feature_data = feature_data.append(n.generators_t.p_max_pu.filter(like=carrier) - .rename(columns=lambda x: x.split(' ')[0] + ' ' + x.split(' ')[1]))[buses_i] + gen_i = n.generators.query("carrier == @carrier").index + attach = n.generators_t.p_max_pu[gen_i].rename(columns = n.generators.loc[gen_i].bus) + feature_data = pd.concat([feature_data, attach], axis=0)[buses_i] + feature_data = feature_data.T - feature_data.columns = feature_data.columns.astype(str) # timestamp raises error in sklearn>=v1.2 + # timestamp raises error in sklearn >= v1.2: + feature_data.columns = feature_data.columns.astype(str) feature_data = feature_data.fillna(0) @@ -283,16 +275,24 @@ def busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights=None, algori m.links = m.links.query("bus0 in @m.buses.index and bus1 in @m.buses.index") _, labels = csgraph.connected_components(m.adjacency_matrix(), directed=False) + component = pd.Series(labels, index=m.buses.index) component_sizes = component.value_counts() if len(component_sizes)>1: - disconnected_bus = component[component==component_sizes[component_sizes==component_sizes.min()].index[0]].index - neighbor_bus = n.lines.query("bus0 in @disconnected_bus or bus1 in @disconnected_bus").iloc[0][['bus0','bus1']] + disconnected_bus = component[component==component_sizes.index[-1]].index[0] + + neighbor_bus = ( + n.lines.query("bus0 == @disconnected_bus or bus1 == @disconnected_bus") + .iloc[0][['bus0', 'bus1']] + ) new_country = list(set(n.buses.loc[neighbor_bus].country)-set([country]))[0] - logger.info(f"overwriting country `{country}` of bus `{disconnected_bus}` to new country `{new_country}`, " - "because it is disconnected from its inital inter-country transmission grid.") + logger.info( + f"overwriting country `{country}` of bus `{disconnected_bus}` " + f"to new country `{new_country}`, because it is disconnected " + "from its inital inter-country transmission grid." + ) n.buses.at[disconnected_bus, "country"] = new_country return n From 4c1c5e3a4e2a527792cb9a3dcc0b4d5c08d45f29 Mon Sep 17 00:00:00 2001 From: martacki Date: Thu, 17 Mar 2022 17:38:30 +0100 Subject: [PATCH 08/90] suggestions by coroa and fneum --- scripts/cluster_network.py | 42 +++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index f4058e79..003442ff 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -189,28 +189,20 @@ def get_feature_for_hac(n, buses_i=None, feature=None): if feature.split('-')[1] == 'cap': feature_data = pd.DataFrame(index=buses_i, columns=carriers) for carrier in carriers: - try: - # without simpl wildcard (bus names are "X X"): - feature_data[carrier] = (n.generators_t.p_max_pu.filter(like=carrier).mean() - .rename(index=lambda x: x.split(' ')[0])) - except: - # with simpl wildcard (bus names are "X X X"): - feature_data[carrier] = (n.generators_t.p_max_pu.filter(like=carrier).mean() - .rename(index=lambda x: x.split(' ')[0] + ' ' + x.split(' ')[1])) + gen_i = n.generators.query("carrier == @carrier").index + attach = n.generators_t.p_max_pu[gen_i].mean().rename(index = n.generators.loc[gen_i].bus) + feature_data[carrier] = attach if feature.split('-')[1] == 'time': feature_data = pd.DataFrame(columns=buses_i) for carrier in carriers: - try: - # without simpl wildcard (bus names are "X X"): - feature_data = feature_data.append(n.generators_t.p_max_pu.filter(like=carrier) - .rename(columns=lambda x: x.split(' ')[0]))[buses_i] - except: - # with simpl wildcard (bus names are "X X X"): - feature_data = feature_data.append(n.generators_t.p_max_pu.filter(like=carrier) - .rename(columns=lambda x: x.split(' ')[0] + ' ' + x.split(' ')[1]))[buses_i] + gen_i = n.generators.query("carrier == @carrier").index + attach = n.generators_t.p_max_pu[gen_i].rename(columns = n.generators.loc[gen_i].bus) + feature_data = pd.concat([feature_data, attach], axis=0)[buses_i] + feature_data = feature_data.T - feature_data.columns = feature_data.columns.astype(str) # timestamp raises error in sklearn>=v1.2 + # timestamp raises error in sklearn >= v1.2: + feature_data.columns = feature_data.columns.astype(str) feature_data = feature_data.fillna(0) @@ -283,16 +275,24 @@ def busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights=None, algori m.links = m.links.query("bus0 in @m.buses.index and bus1 in @m.buses.index") _, labels = csgraph.connected_components(m.adjacency_matrix(), directed=False) + component = pd.Series(labels, index=m.buses.index) component_sizes = component.value_counts() if len(component_sizes)>1: - disconnected_bus = component[component==component_sizes[component_sizes==component_sizes.min()].index[0]].index - neighbor_bus = n.lines.query("bus0 in @disconnected_bus or bus1 in @disconnected_bus").iloc[0][['bus0','bus1']] + disconnected_bus = component[component==component_sizes.index[-1]].index[0] + + neighbor_bus = ( + n.lines.query("bus0 == @disconnected_bus or bus1 == @disconnected_bus") + .iloc[0][['bus0', 'bus1']] + ) new_country = list(set(n.buses.loc[neighbor_bus].country)-set([country]))[0] - logger.info(f"overwriting country `{country}` of bus `{disconnected_bus}` to new country `{new_country}`, " - "because it is disconnected from its inital inter-country transmission grid.") + logger.info( + f"overwriting country `{country}` of bus `{disconnected_bus}` " + f"to new country `{new_country}`, because it is disconnected " + "from its inital inter-country transmission grid." + ) n.buses.at[disconnected_bus, "country"] = new_country return n From 8cb4c17930909c873d66f003ea41a2e09a924e4f Mon Sep 17 00:00:00 2001 From: martacki Date: Tue, 22 Mar 2022 16:53:05 +0100 Subject: [PATCH 09/90] unify vre treatment for hac clustering for simplify_network and cluster_network --- scripts/simplify_network.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index b28bc4dc..f9ac8ad7 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -351,7 +351,7 @@ def aggregate_to_substations(n, buses_i=None): aggregate_generators_carriers=None, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=1.0, - generator_strategies={'p_nom_max': 'sum'}, + generator_strategies={'p_nom_max': 'sum', 'p_nom_min': 'sum'}, scale_link_capital_costs=False) return clustering.network, busmap @@ -407,21 +407,20 @@ if __name__ == "__main__": n, substation_map = aggregate_to_substations(n) busmaps.append(substation_map) - # treatment of outliers (nodes without a profile for considered carrier) for "cluster_network" - if snakemake.config.get("clustering", {}).get("cluster_network", {}).get("algorithm", "hac") == "hac": + # treatment of outliers (nodes without a profile for considered carrier): + # all nodes that have no profile of the given carrier are being aggregated to closest neighbor + if ( + snakemake.config.get("clustering", {}).get("cluster_network", {}).get("algorithm", "hac") == "hac" or + cluster_config.get("algorithm", "hac") == "hac" + ): carriers = cluster_config.get("feature", "solar+onwind-time").split('-')[0].split('+') - buses_i = list(set(n.buses.index)-set(n.generators.query("carrier in @carriers").bus)) - n, busmap_hac = aggregate_to_substations(n, buses_i) - busmaps.append(busmap_hac) - - if snakemake.wildcards.simpl: - # treatment of outliers (nodes without a profile for a considered carrier) for "simplify" - if cluster_config.get("algorithm", "hac") == "hac": - carriers = cluster_config.get("feature", "solar+onwind-time").split('-')[0].split('+') - buses_i = list(set(n.buses.index)-set(n.generators.query("carrier in @carriers").bus)) + for carrier in carriers: + buses_i = list(set(n.buses.index)-set(n.generators.query("carrier == @carrier").bus)) + logger.info(f'clustering preparaton (hac): aggregating {len(buses_i)} buses of type {carrier}.') n, busmap_hac = aggregate_to_substations(n, buses_i) busmaps.append(busmap_hac) - # conduct clustering + + if snakemake.wildcards.simpl: n, cluster_map = cluster(n, int(snakemake.wildcards.simpl), snakemake.config, algorithm=cluster_config.get('algorithm', 'hac'), feature=cluster_config.get('feature', None)) From 7f29c31abe6cf133f28f4b4558cccb7f20899d72 Mon Sep 17 00:00:00 2001 From: martacki Date: Thu, 24 Mar 2022 13:17:01 +0100 Subject: [PATCH 10/90] .copy() shortcut --- scripts/cluster_network.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 9c6d541f..f659f8a8 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -269,10 +269,7 @@ def busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights=None, algori # overwrite country of nodes that are disconnected from their country-topology for country in n.buses.country.unique(): - m = n.copy() - m.buses = m.buses.query("country in @country") - m.lines = m.lines.query("bus0 in @m.buses.index and bus1 in @m.buses.index") - m.links = m.links.query("bus0 in @m.buses.index and bus1 in @m.buses.index") + m = n[n.buses.country ==country].copy() _, labels = csgraph.connected_components(m.adjacency_matrix(), directed=False) From 7bc9b8012c1fe1c747d6386656bade8be70d7a5e Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 4 Apr 2022 18:07:48 +0200 Subject: [PATCH 11/90] powerplants: update to ppm >= v0.5.1 --- envs/environment.yaml | 2 +- scripts/build_powerplants.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 3c69b77b..795aa334 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -24,7 +24,6 @@ dependencies: - yaml - pytables - lxml - - powerplantmatching>=0.4.8 - numpy - pandas - geopandas @@ -57,3 +56,4 @@ dependencies: - pip: - vresutils>=0.3.1 - tsam>=1.1.0 + - powerplantmatching>=0.5.1 diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index 764028d1..e18232b8 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -104,7 +104,7 @@ if __name__ == "__main__": countries = n.buses.country.unique() ppl = (pm.powerplants(from_url=True) - .powerplant.fill_missing_decommyears() + .powerplant.fill_missing_decommissioning_years() .powerplant.convert_country_to_alpha2() .query('Fueltype not in ["Solar", "Wind"] and Country in @countries') .replace({'Technology': {'Steam Turbine': 'OCGT'}}) From ade22bf4f003b774185e4a4ad123a6c33913bb0e Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 4 Apr 2022 19:03:09 +0200 Subject: [PATCH 12/90] add existing wind and solar capacities based on IRENASTATS --- config.default.yaml | 18 +++++++++++---- doc/release_notes.rst | 20 ++++++++++++++++ envs/environment.yaml | 2 +- scripts/add_electricity.py | 47 ++++++++++++++++++++------------------ 4 files changed, 60 insertions(+), 27 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index d2bf6159..f952f5b7 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -58,10 +58,20 @@ electricity: conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] renewable_capacities_from_OPSD: [] # onwind, offwind, solar - # estimate_renewable_capacities_from_capacity_stats: - # # Wind is the Fueltype in ppm.data.Capacity_stats, onwind, offwind-{ac,dc} the carrier in PyPSA-Eur - # Wind: [onwind, offwind-ac, offwind-dc] - # Solar: [solar] + estimate_renewable_capacities: + # Renewable capacities are based on existing capacities reported by IRENA + + # Reference year, any of 2000 to 2020 + year: 2020 + # Artificially limit maximum capacities to factor * (IRENA capacities), + # i.e. 110% of 's capacities => expansion_limit: 1.1 + # false: Use estimated renewable potentials determine by the workflow + expansion_limit: false + technology_mapping: + # Wind is the Fueltype in ppm.data.Capacity_stats, onwind, offwind-{ac,dc} the carrier in PyPSA-Eur + Offshore: [offwind-ac, offwind-dc] + Onshore: [onwind] + PV: [solar] atlite: nprocesses: 4 diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 3f131dc0..9b012bf8 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -7,6 +7,26 @@ Release Notes ########################################## +Energy Security Release (April 2022) +==================================== + +**New Features and Changes** + +* Added existing renewable capacities for all countries based on IRENA statistics (IRENASTAT) using new ``powerplantmatching`` version: + * Configuration of reference year for capacities can be configured (default: ``2020``) + * Uniform expansion limit of renewable build-up based on existing capacities can be configured using ``expansion_limit`` option + (default: ``false``; limited to determined renewable potentials) + * Distribution of country-level capacities proportional to maximum annual energy yield for each bus region + * This functionality was previously using OPSD data. + * The corresponding ``config`` entries changed, cf. ``config.default.yaml``: + * old: ``estimate_renewable_capacities_from_capacity_stats`` + * new: ``estimate_renewable_capacities`` + + +**Bugs and Compatibility** + +* ``powerplantmatching>=0.5.1`` is now required for ``IRENASTATS``. + Synchronisation Release - Ukraine and Moldova (17th March 2022) =============================================================== diff --git a/envs/environment.yaml b/envs/environment.yaml index 3c69b77b..d02b0018 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -24,7 +24,7 @@ dependencies: - yaml - pytables - lxml - - powerplantmatching>=0.4.8 + - powerplantmatching>=0.5.1 - numpy - pandas - geopandas diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index ad932cd8..f468bab3 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: : 2017-2020 The PyPSA-Eur Authors +# SPDX-FileCopyrightText: : 2017-2022 The PyPSA-Eur Authors # # SPDX-License-Identifier: MIT @@ -491,29 +491,29 @@ def attach_OPSD_renewables(n, techs): -def estimate_renewable_capacities(n, tech_map): +def estimate_renewable_capacities(n, config): - if len(tech_map) == 0: return - - capacities = (pm.data.Capacity_stats().powerplant.convert_country_to_alpha2() - [lambda df: df.Energy_Source_Level_2] - .set_index(['Fueltype', 'Country']).sort_index()) - - countries = n.buses.country.unique() + if not config["electricity"]["estimate_renewable_capacities"]: return + + year = config["electricity"]["estimate_renewable_capacities"]["year"] + tech_map = config["electricity"]["estimate_renewable_capacities"]["technology_mapping"] + tech_keys = list(tech_map.keys()) + countries = config["countries"] + expansion_limit = config["electricity"]["estimate_renewable_capacities"]["expansion_limit"] if len(countries) == 0: return + if len(tech_map) == 0: return - logger.info('heuristics applied to distribute renewable capacities [MW] \n{}' - .format(capacities.query('Fueltype in @tech_map.keys() and Capacity >= 0.1') - .groupby('Country').agg({'Capacity': 'sum'}))) + capacities = pm.data.IRENASTAT().powerplant.convert_country_to_alpha2() + capacities = capacities.query("Year == @year and Technology in @tech_keys and Country in @countries") + capacities = capacities.groupby(["Technology", "Country"]).Capacity.sum() - for ppm_fueltype, techs in tech_map.items(): - tech_capacities = capacities.loc[ppm_fueltype, 'Capacity']\ - .reindex(countries, fill_value=0.) - #tech_i = n.generators.query('carrier in @techs').index - tech_i = (n.generators.query('carrier in @techs') - [n.generators.query('carrier in @techs') - .bus.map(n.buses.country).isin(countries)].index) + logger.info(f"Heuristics applied to distribute renewable capacities [MW] " + f"{capacities.groupby('Country').sum()}") + + for ppm_technology, techs in tech_map.items(): + tech_capacities = capacities.loc[ppm_technology].reindex(countries, fill_value=0.) + tech_i = n.generators.query('carrier in @techs').index n.generators.loc[tech_i, 'p_nom'] = ( (n.generators_t.p_max_pu[tech_i].mean() * n.generators.loc[tech_i, 'p_nom_max']) # maximal yearly generation @@ -522,6 +522,11 @@ def estimate_renewable_capacities(n, tech_map): .where(lambda s: s>0.1, 0.)) # only capacities above 100kW n.generators.loc[tech_i, 'p_nom_min'] = n.generators.loc[tech_i, 'p_nom'] + if expansion_limit: + assert np.isscalar(expansion_limit) + logger.info(f"Reducing capacity expansion limit to {expansion_limit*100:.2f}% of installed capacity.") + n.generators.loc[tech_i, 'p_nom_max'] = float(expansion_limit) * n.generators.loc[tech_i, 'p_nom_min'] + def add_nice_carrier_names(n, config): carrier_i = n.carriers.index @@ -565,11 +570,9 @@ if __name__ == "__main__": carriers = snakemake.config['electricity']['extendable_carriers']['Generator'] attach_extendable_generators(n, costs, ppl, carriers) - tech_map = snakemake.config['electricity'].get('estimate_renewable_capacities_from_capacity_stats', {}) - estimate_renewable_capacities(n, tech_map) + estimate_renewable_capacities(n, snakemake.config) techs = snakemake.config['electricity'].get('renewable_capacities_from_OPSD', []) attach_OPSD_renewables(n, techs) - update_p_nom_max(n) add_nice_carrier_names(n, snakemake.config) From f878faac73b9ef61d0a4c146c8ac6be0a990dd7b Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 5 Apr 2022 08:02:44 +0200 Subject: [PATCH 13/90] add_electricity: allow estimate_renewable_capacities to be commented out --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index f468bab3..961b87dc 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -493,7 +493,7 @@ def attach_OPSD_renewables(n, techs): def estimate_renewable_capacities(n, config): - if not config["electricity"]["estimate_renewable_capacities"]: return + if not config["electricity"].get("estimate_renewable_capacities"): return year = config["electricity"]["estimate_renewable_capacities"]["year"] tech_map = config["electricity"]["estimate_renewable_capacities"]["technology_mapping"] From 1fd6a685ab636ca918eb58a5e3dbac175d66ad68 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 5 Apr 2022 15:12:01 +0200 Subject: [PATCH 14/90] powerplants: filter out powerplants with shut down date < 2021 --- config.default.yaml | 6 ++++-- doc/release_notes.rst | 1 + scripts/build_powerplants.py | 13 +++++++------ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index f952f5b7..4cd0eadc 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -53,8 +53,10 @@ electricity: battery: 6 H2: 168 - powerplants_filter: false # use pandas query strings here, e.g. Country not in ['Germany'] - custom_powerplants: false # use pandas query strings here, e.g. Country in ['Germany'] + # use pandas query strings here, e.g. Country not in ['Germany'] + powerplants_filter: (DateOut >= 2021 or DateOut != DateOut) + # use pandas query strings here, e.g. Country in ['Germany'] + custom_powerplants: false conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] renewable_capacities_from_OPSD: [] # onwind, offwind, solar diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 9b012bf8..4ef5e956 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -22,6 +22,7 @@ Energy Security Release (April 2022) * old: ``estimate_renewable_capacities_from_capacity_stats`` * new: ``estimate_renewable_capacities`` +* The powerplants that have been shut down before 2021 are filtered out. **Bugs and Compatibility** diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index e18232b8..9a7c9e23 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -94,6 +94,10 @@ def add_custom_powerplants(ppl, custom_powerplants, custom_ppl_query=False): return pd.concat([ppl, add_ppls], sort=False, ignore_index=True, verify_integrity=True) +def replace_natural_gas_by_technology(df): + return df.Fueltype.where(df.Fueltype != 'Natural Gas', df.Technology) + + if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake @@ -103,16 +107,13 @@ if __name__ == "__main__": n = pypsa.Network(snakemake.input.base_network) countries = n.buses.country.unique() + ppl = (pm.powerplants(from_url=True) .powerplant.fill_missing_decommissioning_years() .powerplant.convert_country_to_alpha2() .query('Fueltype not in ["Solar", "Wind"] and Country in @countries') - .replace({'Technology': {'Steam Turbine': 'OCGT'}}) - .assign(Fueltype=lambda df: ( - df.Fueltype - .where(df.Fueltype != 'Natural Gas', - df.Technology.replace('Steam Turbine', - 'OCGT').fillna('OCGT'))))) + .replace({'Technology': {'Steam Turbine': 'OCGT', "Combustion Engine": "OCGT"}}) + .assign(Fueltype=replace_natural_gas_by_technology)) ppl_query = snakemake.config['electricity']['powerplants_filter'] if isinstance(ppl_query, str): From 7a52b6bc455e160b9b32ca95400e5b882da32e0e Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 28 Mar 2022 12:02:08 +0200 Subject: [PATCH 15/90] resolve cherry merge conflict 1 --- Snakefile | 4 ++-- scripts/build_hydro_profile.py | 29 ++++++++++++++++++++++++++--- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/Snakefile b/Snakefile index 7678a401..f3a894fc 100644 --- a/Snakefile +++ b/Snakefile @@ -50,7 +50,7 @@ if config['enable'].get('prepare_links_p_nom', False): datafiles = ['ch_cantons.csv', 'je-e-21.03.02.xls', - 'eez/World_EEZ_v8_2014.shp', 'EIA_hydro_generation_2000_2014.csv', + 'eez/World_EEZ_v8_2014.shp', 'hydro_capacities.csv', 'naturalearth/ne_10m_admin_0_countries.shp', 'NUTS_2013_60M_SH/data/NUTS_RG_60M_2013.shp', 'nama_10r_3popgdp.tsv.gz', 'nama_10r_3gdp.tsv.gz', 'corine/g250_clc06_V18_5.tif'] @@ -208,7 +208,7 @@ rule build_renewable_profiles: rule build_hydro_profile: input: country_shapes='resources/country_shapes.geojson', - eia_hydro_generation='data/bundle/EIA_hydro_generation_2000_2014.csv', + eia_hydro_generation='data/eia_hydro_annual_generation.csv', cutout=f"cutouts/{config['renewable']['hydro']['cutout']}.nc" if "hydro" in config["renewable"] else "config['renewable']['hydro']['cutout'] not configured", output: 'resources/profile_hydro.nc' log: "logs/build_hydro_profile.log" diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 74efc2ef..0fb20b06 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -64,7 +64,29 @@ from _helpers import configure_logging import atlite import geopandas as gpd -from vresutils import hydro as vhydro +import pandas as pd + +import country_converter as coco +cc = coco.CountryConverter() + + +def get_eia_annual_hydro_generation(fn, countries): + + # in billion kWh/a = TWh/a + df = pd.read_csv(fn, skiprows=2, index_col=1, na_values=[u' ','--']).iloc[1:, 1:] + df.index = df.index.str.strip() + + df.loc["Germany"] = df.filter(like='Germany', axis=0).sum() + df.loc["Serbia"] += df.loc["Kosovo"] + df = df.loc[~df.index.str.contains('Former')] + + df.index = cc.convert(df.index, to='iso2') + df.index.name = 'countries' + + df = df.T[countries] * 1e6 # in MWh/a + + return df + logger = logging.getLogger(__name__) @@ -82,8 +104,9 @@ if __name__ == "__main__": .set_index('name')['geometry'].reindex(countries)) country_shapes.index.name = 'countries' - eia_stats = vhydro.get_eia_annual_hydro_generation( - snakemake.input.eia_hydro_generation).reindex(columns=countries) + fn = snakemake.input.eia_hydro_generation + eia_stats = get_eia_annual_hydro_generation(fn, countries) + inflow = cutout.runoff(shapes=country_shapes, smooth=True, lower_threshold_quantile=True, From 998761ec8851c3fae06c8e9c738cba7d31f80ed5 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 28 Mar 2022 12:03:23 +0200 Subject: [PATCH 16/90] build_hydro: add new EIA hydro dataset --- data/eia_hydro_annual_generation.csv | 50 ++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 data/eia_hydro_annual_generation.csv diff --git a/data/eia_hydro_annual_generation.csv b/data/eia_hydro_annual_generation.csv new file mode 100644 index 00000000..cb1ae12f --- /dev/null +++ b/data/eia_hydro_annual_generation.csv @@ -0,0 +1,50 @@ +https://www.eia.gov/international/data/world/electricity/electricity-generation?pd=2&p=000000000000000000000000000000g&u=1&f=A&v=mapbubble&a=-&i=none&vo=value&t=R&g=000000000000002&l=73-1028i008017kg6368g80a4k000e0ag00gg0004g8g0ho00g000400008&s=315532800000&e=1577836800000&ev=false& +Report generated on: 03-28-2022 11:20:48 +"API","","1980","1981","1982","1983","1984","1985","1986","1987","1988","1989","1990","1991","1992","1993","1994","1995","1996","1997","1998","1999","2000","2001","2002","2003","2004","2005","2006","2007","2008","2009","2010","2011","2012","2013","2014","2015","2016","2017","2018","2019","2020" +"","hydroelectricity net generation (billion kWh)","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","" +"INTL.33-12-EURO-BKWH.A"," Europe","458.018","464.155","459.881","473.685","481.241","476.739","459.535","491.085","534.517","465.365","474.466","475.47","509.041","526.448","531.815","543.743","529.114164","543.845616","562.441501","569.308453","591.206662","587.371195","541.542535","506.19703","544.536443","545.176179","537.335934","540.934407","567.557921","564.244482","619.96477","543.05273","600.46622","631.86431","619.59229","615.53013","629.98906","562.59258","619.31106","610.62616","670.925" +"INTL.33-12-ALB-BKWH.A"," Albania","2.919","3.018","3.093","3.167","3.241","3.315","3.365","3.979","3.713","3.846","2.82","3.483","3.187","3.281","3.733","4.162","5.669","4.978","4.872","5.231","4.548","3.519","3.477","5.117","5.411","5.319","4.951","2.76","3.759","5.201","7.49133","4.09068","4.67775","6.88941","4.67676","5.83605","7.70418","4.47975","8.46648","5.15394","5.281" +"INTL.33-12-AUT-BKWH.A"," Austria","28.501","30.008","29.893","29.577","28.384","30.288","30.496","25.401","35.151","34.641","31.179","31.112","34.483","36.336","35.349","36.696","33.874","35.744","36.792","40.292","41.418","40.05","39.825","32.883","36.394","36.31","35.48","36.732","37.969","40.487","36.466","32.511","41.862","40.138","39.001","35.255","37.954","36.462","35.73","40.43655","45.344" +"INTL.33-12-BEL-BKWH.A"," Belgium","0.274","0.377","0.325","0.331","0.348","0.282","0.339","0.425","0.354","0.3","0.263","0.226","0.338","0.252","0.342","0.335","0.237","0.30195","0.38511","0.338","0.455","0.437","0.356","0.245","0.314","0.285","0.355","0.385","0.406","0.325","0.298","0.193","0.353","0.376","0.289","0.314","0.367","0.268","0.311","0.108","1.29" +"INTL.33-12-BIH-BKWH.A"," Bosnia and Herzegovina","--","--","--","--","--","--","--","--","--","--","--","--","3.374","2.343","3.424","3.607","5.104","4.608","4.511","5.477","5.043","5.129","5.215","4.456","5.919","5.938","5.798","3.961","4.818","6.177","7.946","4.343","4.173","7.164","5.876","5.495","5.585","3.7521","6.35382","6.02019","6.1" +"INTL.33-12-BGR-BKWH.A"," Bulgaria","3.674","3.58","3.018","3.318","3.226","2.214","2.302","2.512","2.569","2.662","1.859","2.417","2.042","1.923","1.453","2.291","2.89","2.726","3.066","2.725","2.646","1.72","2.172","2.999","3.136","4.294","4.196","2.845","2.796","3.435","4.98168","2.84328","3.14622","3.99564","4.55598","5.59845","3.8412","2.79972","5.09553","3.34917","3.37" +"INTL.33-12-HRV-BKWH.A"," Croatia","--","--","--","--","--","--","--","--","--","--","--","--","4.298","4.302","4.881","5.212","7.156","5.234","5.403","6.524","5.794","6.482","5.311","4.827","6.888","6.27","5.94","4.194","5.164","6.663","9.035","4.983","4.789","8.536","8.917","6.327","6.784","5.255","7.62399","5.87268","3.4" +"INTL.33-12-CYP-BKWH.A"," Cyprus","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0" +"INTL.33-12-CZE-BKWH.A"," Czech Republic","--","--","--","--","--","--","--","--","--","--","--","--","--","1.355","1.445","1.982","1.949","1.68201","1.382","1.664","1.7404","2.033","2.467","1.369","1.999","2.356","2.525","2.068","2.004","2.405","2.775","1.95","2.107","2.704","1.909","1.779","1.983","1.852","1.615","1.98792","3.4" +"INTL.33-12-DNK-BKWH.A"," Denmark","0.03","0.031","0.028","0.036","0.028","0.027","0.029","0.029","0.032","0.027","0.027","0.026","0.028","0.027","0.033","0.03","0.019","0.019","0.02673","0.031","0.03","0.028","0.032","0.021","0.027","0.023","0.023","0.028","0.026","0.019","0.021","0.017","0.017","0.013","0.015","0.018","0.019","0.018","0.015","0.01584","0.02" +"INTL.33-12-EST-BKWH.A"," Estonia","--","--","--","--","--","--","--","--","--","--","--","--","0.001","0.001","0.003","0.002","0.002","0.003","0.004","0.004","0.005","0.007","0.006","0.013","0.022","0.022","0.014","0.021","0.028","0.032","0.027","0.03","0.042","0.026","0.027","0.027","0.035","0.026","0.015","0.01881","0.04" +"INTL.33-12-FRO-BKWH.A"," Faroe Islands","0.049","0.049","0.049","0.049","0.049","0.049","0.049","0.049","0.062","0.071","0.074","0.074","0.083","0.073","0.075","0.075","0.069564","0.075066","0.076501","0.069453","0.075262","0.075195","0.095535","0.08483","0.093443","0.097986","0.099934","0.103407","0.094921","0.091482","0.06676","0.092","0.099","0.091","0.121","0.132","0.105","0.11","0.107","0.102","0.11" +"INTL.33-12-FIN-BKWH.A"," Finland","10.115","13.518","12.958","13.445","13.115","12.211","12.266","13.658","13.229","12.9","10.75","13.065","14.956","13.341","11.669","12.796","11.742","12.11958","14.9","12.652","14.513","13.073","10.668","9.495","14.919","13.646","11.379","14.035","16.941","12.559","12.743","12.278","16.667","12.672","13.24","16.584","15.634","14.61","13.137","12.31461","15.56" +"INTL.33-12-CSK-BKWH.A"," Former Czechoslovakia","4.8","4.2","3.7","3.9","3.2","4.3","4","4.853","4.355","4.229","3.919","3.119","3.602","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-12-SCG-BKWH.A"," Former Serbia and Montenegro","--","--","--","--","--","--","--","--","--","--","--","--","11.23","10.395","11.016","12.071","14.266","12.636","12.763","13.243","11.88","12.326","11.633","9.752","11.01","11.912","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-12-YUG-BKWH.A"," Former Yugoslavia","27.868","25.044","23.295","21.623","25.645","24.363","27.474","25.98","25.612","23.256","19.601","18.929","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-12-FRA-BKWH.A"," France","68.253","70.358","68.6","67.515","64.01","60.248","60.953","68.623","73.952","45.744","52.796","56.277","68.313","64.3","78.057","72.196","64.43","63.151","61.479","71.832","66.466","73.888","59.992","58.567","59.276","50.965","55.741","57.029","63.017","56.428","61.945","45.184","59.099","71.042","62.993","54.876","60.094","49.389","64.485","56.98242","64.84" +"INTL.33-12-DEU-BKWH.A"," Germany","--","--","--","--","--","--","--","--","--","--","--","14.742","17.223","17.699","19.731","21.562","21.737","17.18343","17.044","19.451","21.515","22.506","22.893","19.071","20.866","19.442","19.808","20.957","20.239","18.841","20.678","17.323","21.331","22.66","19.31","18.664","20.214","19.985","17.815","19.86039","24.75" +"INTL.33-12-DDR-BKWH.A"," Germany, East","1.658","1.718","1.748","1.683","1.748","1.758","1.767","1.726","1.719","1.551","1.389","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-12-DEUW-BKWH.A"," Germany, West","17.125","17.889","17.694","16.713","16.434","15.354","16.526","18.36","18.128","16.482","15.769","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--" +"INTL.33-12-GIB-BKWH.A"," Gibraltar","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0" +"INTL.33-12-GRC-BKWH.A"," Greece","3.396","3.398","3.551","2.331","2.852","2.792","3.222","2.768","2.354","1.888","1.751","3.068","2.181","2.26","2.573","3.494","4.305","3.84318","3.68","4.546","3.656","2.076","2.772","4.718","4.625","4.967","5.806","2.565","3.279","5.32","7.431","3.998","4.387","6.337","4.464","5.782","5.543","3.962","5.035","3.9798","3.43" +"INTL.33-12-HUN-BKWH.A"," Hungary","0.111","0.166","0.158","0.153","0.179","0.153","0.152","0.167","0.167","0.156","0.176","0.192","0.156","0.164","0.159","0.161","0.205","0.21384","0.15345","0.179","0.176","0.184","0.192","0.169","0.203","0.2","0.184","0.208","0.211","0.226","0.184","0.216","0.206","0.208","0.294","0.227","0.253","0.214","0.216","0.21681","0.24" +"INTL.33-12-ISL-BKWH.A"," Iceland","3.053","3.085","3.407","3.588","3.738","3.667","3.846","3.918","4.169","4.217","4.162","4.162","4.267","4.421","4.47","4.635","4.724","5.15493","5.565","5.987","6.292","6.512","6.907","7.017","7.063","6.949","7.22","8.31","12.303","12.156","12.51","12.382","12.214","12.747","12.554","13.541","13.092","13.892","13.679","13.32441","12.46" +"INTL.33-12-IRL-BKWH.A"," Ireland","0.833","0.855","0.792","0.776","0.68","0.824","0.91","0.673","0.862","0.684","0.69","0.738","0.809","0.757","0.911","0.706","0.715","0.67122","0.907","0.838","0.838","0.59","0.903","0.592","0.624","0.625","0.717","0.66","0.959","0.893","0.593","0.699","0.795","0.593","0.701","0.798","0.674","0.685","0.687","0.87813","1.21" +"INTL.33-12-ITA-BKWH.A"," Italy","44.997","42.782","41.216","40.96","41.923","40.616","40.626","39.05","40.205","33.647","31.31","41.817","41.778","41.011","44.212","37.404","41.617","41.18697","40.808","44.911","43.763","46.343","39.125","33.303","41.915","35.706","36.624","32.488","41.207","48.647","50.506","45.36477","41.45625","52.24626","57.95955","45.08163","42.00768","35.83701","48.29913","45.31824","47.72" +"INTL.33-12-XKS-BKWH.A"," Kosovo","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","0.075","0.119","0.154","0.104","0.095","0.142","0.149","0.139","0.243","0.177","0.27027","0.2079","0.26" +"INTL.33-12-LVA-BKWH.A"," Latvia","--","--","--","--","--","--","--","--","--","--","--","--","2.498","2.846","3.272","2.908","1.841","2.922","2.99","2.729","2.791","2.805","2.438","2.243","3.078","3.293","2.671","2.706","3.078","3.422","3.488","2.857","3.677","2.838","1.953","1.841","2.523","4.356","2.417","2.08692","2.59" +"INTL.33-12-LTU-BKWH.A"," Lithuania","--","--","--","--","--","--","--","--","--","--","--","--","0.308","0.389","0.447","0.369","0.323","0.291","0.413","0.409","0.336","0.322","0.35","0.323","0.417","0.446193","0.393","0.417","0.398","0.42","0.535","0.475","0.419","0.516","0.395","0.346","0.45","0.597","0.427","0.34254","1.06" +"INTL.33-12-LUX-BKWH.A"," Luxembourg","0.086","0.095","0.084","0.083","0.088","0.071","0.084","0.101","0.097","0.072","0.07","0.083","0.069","0.066","0.117","0.087","0.059","0.082","0.114","0.084","0.119","0.117","0.098","0.078","0.103","0.093","0.11","0.116","0.131","0.105","0.104","0.061","0.095","0.114","0.104","0.095","0.111","0.082","0.089","0.10593","1.09" +"INTL.33-12-MLT-BKWH.A"," Malta","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0" +"INTL.33-12-MNE-BKWH.A"," Montenegro","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","1.733","1.271","1.524","2.05","2.723","1.192","1.462","2.479","1.734","1.476","1.825","1.014","2.09187","1.78","1.8" +"INTL.33-12-NLD-BKWH.A"," Netherlands","0","0","0","0","0","0.003","0.003","0.001","0.002","0.037","0.119","0.079","0.119","0.091","0.1","0.087","0.079","0.09108","0.111","0.089","0.141","0.116","0.109","0.071","0.094","0.087","0.105","0.106","0.101","0.097","0.105","0.057","0.104","0.114","0.112","0.093","0.1","0.061","0.072","0.07326","0.05" +"INTL.33-12-MKD-BKWH.A"," North Macedonia","--","--","--","--","--","--","--","--","--","--","--","--","0.817","0.517","0.696","0.793","0.842","0.891","1.072","1.375","1.158","0.62","0.749","1.36","1.467","1.477","1.634","1","0.832","1.257","2.407","1.419","1.031","1.568","1.195","1.846","1.878","1.099","1.773","1.15236","1.24" +"INTL.33-12-NOR-BKWH.A"," Norway","82.717","91.876","91.507","104.704","104.895","101.464","95.321","102.341","107.919","117.369","119.933","109.032","115.505","118.024","110.398","120.315","102.823","108.677","114.546","120.237","140.4","119.258","128.078","104.425","107.693","134.331","118.175","132.319","137.654","124.03","116.257","119.78","141.189","127.551","134.844","136.662","142.244","141.651","138.202","123.66288","141.69" +"INTL.33-12-POL-BKWH.A"," Poland","2.326","2.116","1.528","1.658","1.394","1.833","1.534","1.644","1.775","1.593","1.403","1.411","1.492","1.473","1.716","1.868","1.912","1.941","2.286","2.133","2.085","2.302","2.256","1.654","2.06","2.179","2.022","2.328","2.13","2.351","2.9","2.313","2.02","2.421","2.165","1.814","2.117","2.552","1.949","1.93842","2.93" +"INTL.33-12-PRT-BKWH.A"," Portugal","7.873","4.934","6.82","7.897","9.609","10.512","8.364","9.005","12.037","5.72","9.065","8.952","4.599","8.453","10.551","8.26","14.613","12.97395","12.853","7.213","11.21","13.894","7.722","15.566","9.77","4.684","10.892","9.991","6.73","8.201","15.954","11.423","5.589","13.652","15.471","8.615","15.608","5.79","12.316","8.6526","13.96" +"INTL.33-12-ROU-BKWH.A"," Romania","12.506","12.605","11.731","9.934","11.208","11.772","10.688","11.084","13.479","12.497","10.87","14.107","11.583","12.64","12.916","16.526","15.597","17.334","18.69","18.107","14.63","14.774","15.886","13.126","16.348","20.005","18.172","15.806","17.023","15.379","19.684","14.581","11.945","14.807","18.618","16.467","17.848","14.349","17.48736","15.65289","15.53" +"INTL.33-12-SRB-BKWH.A"," Serbia","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","--","10.855","9.937","9.468","10.436","11.772","8.58","9.193","10.101","10.893","9.979","10.684","9.061","10.53261","10.07028","9.66" +"INTL.33-12-SVK-BKWH.A"," Slovakia","--","--","--","--","--","--","--","--","--","--","--","--","--","3.432","4.311","4.831","4.185","4.023","4.224","4.429","4.569","4.878","5.215","3.4452","4.059","4.592","4.355","4.406","4","4.324","5.184","3.211","3.687","4.329","3.762","3.701","4.302","4.321","3.506","4.27383","4.67" +"INTL.33-12-SVN-BKWH.A"," Slovenia","--","--","--","--","--","--","--","--","--","--","--","--","3.379","2.974","3.348","3.187","3.616","3.046","3.4","3.684","3.771","3.741","3.265","2.916","4.033","3.426","3.555","3.233","3.978","4.666","4.452","3.506","3.841","4.562","6.011","3.75","4.443","3.814","4.643","4.43421","5.24" +"INTL.33-12-ESP-BKWH.A"," Spain","29.16","21.64","25.99","26.696","31.088","30.895","26.105","27.016","34.76","19.046","25.16","27.01","18.731","24.133","27.898","22.881","39.404","34.43","33.665","22.634","29.274","40.617","22.691","40.643","31.359","18.209","25.699","27.036","23.13","26.147","41.576","30.07","20.192","36.45","38.815","27.656","35.77","18.007","33.743","24.23025","33.34" +"INTL.33-12-SWE-BKWH.A"," Sweden","58.133","59.006","54.369","62.801","67.106","70.095","60.134","70.95","69.016","70.911","71.778","62.603","73.588","73.905","58.508","67.421","51.2226","68.365","74.25","70.974","77.798","78.269","65.696","53.005","59.522","72.075","61.106","65.497","68.378","65.193","66.279","66.047","78.333","60.81","63.227","74.734","61.645","64.651","61.79","64.46583","71.6" +"INTL.33-12-CHE-BKWH.A"," Switzerland","32.481","35.13","35.974","35.069","29.871","31.731","32.576","34.328","35.437","29.477","29.497","31.756","32.373","35.416","38.678","34.817","28.458","33.70257","33.136","39.604","36.466","40.895","34.862","34.471","33.411","30.914","30.649","34.898","35.676","35.366","35.704","32.069","38.218","38.08","37.659","37.879","34.281","33.754","34.637","37.6596","40.62" +"INTL.33-12-TUR-BKWH.A"," Turkey","11.159","12.308","13.81","11.13","13.19","11.822","11.637","18.314","28.447","17.61","22.917","22.456","26.302","33.611","30.28","35.186","40.07","39.41784","41.80671","34.33","30.57","23.77","33.346","34.977","45.623","39.165","43.802","35.492","32.937","35.598","51.423","51.155","56.669","58.225","39.75","65.856","66.686","57.824","59.49","87.99714","77.39" +"INTL.33-12-GBR-BKWH.A"," United Kingdom","3.921","4.369","4.543","4.548","3.992","4.08","4.767","4.13","4.915","4.732","5.119","4.534","5.329","4.237","5.043","4.79","3.359","4.127","5.067","5.283","5.035","4.015","4.74","3.195","4.795","4.873","4.547","5.026","5.094","5.178","3.566","5.655","5.286","4.667","5.832","6.246","5.342","5.836","5.189","5.89941","7.64" \ No newline at end of file From 9812e64e8228be3e297fcf5e959e2bf31f526d35 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 29 Mar 2022 09:17:28 +0200 Subject: [PATCH 17/90] resolve cherry merge conflict 2 --- envs/environment.yaml | 2 +- scripts/build_hydro_profile.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 795aa334..9772d882 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -37,6 +37,7 @@ dependencies: - matplotlib - proj - fiona <= 1.18.20 # Till issue https://github.com/Toblerity/Fiona/issues/1085 is not solved + - country_converter # Keep in conda environment when calling ipython - ipython @@ -50,7 +51,6 @@ dependencies: - geopy - tqdm - pytz - - country_converter - tabula-py - pip: diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 0fb20b06..4add4c85 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -79,6 +79,7 @@ def get_eia_annual_hydro_generation(fn, countries): df.loc["Germany"] = df.filter(like='Germany', axis=0).sum() df.loc["Serbia"] += df.loc["Kosovo"] df = df.loc[~df.index.str.contains('Former')] + df.drop(["Europe", "Germany, West", "Germany, East"], inplace=True) df.index = cc.convert(df.index, to='iso2') df.index.name = 'countries' From 51dffbefa393a2498b8d2fd28aaaf63a051ec1f0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 5 Apr 2022 16:59:51 +0200 Subject: [PATCH 18/90] add gas usage limit constraint --- config.default.yaml | 1 + scripts/prepare_network.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/config.default.yaml b/config.default.yaml index 4cd0eadc..128c7bac 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -39,6 +39,7 @@ enable: electricity: voltages: [220., 300., 380.] + gaslimit: false co2limit: 7.75e+7 # 0.05 * 3.1e9*0.5 co2base: 1.487e+9 agg_p_nom_limits: data/agg_p_nom_minmax.csv diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 206e220b..27aacacf 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -77,6 +77,16 @@ def add_co2limit(n, co2limit, Nyears=1.): constant=co2limit * Nyears) +def add_gaslimit(n, gaslimit, Nyears=1.): + + sel = n.carriers.index.intersection(["OCGT", "CCGT", "CHP"]) + n.carriers.loc[sel, "gas_usage"] = 1. + + n.add("GlobalConstraint", "GasLimit", + carrier_attribute="gas_usage", sense="<=", + constant=gaslimit * Nyears) + + def add_emission_prices(n, emission_prices={'co2': 0.}, exclude_co2=False): if exclude_co2: emission_prices.pop('co2') ep = (pd.Series(emission_prices).rename(lambda x: x+'_emissions') * @@ -237,6 +247,10 @@ if __name__ == "__main__": add_co2limit(n, snakemake.config['electricity']['co2limit'], Nyears) break + gaslimit = snakemake.config["electricity"].get("gaslimit") + if gaslimit: + add_gaslimit(n, gaslimit, Nyears) + for o in opts: oo = o.split("+") suptechs = map(lambda c: c.split("-", 2)[0], n.carriers.index) From c4bb470b933237a114efcaf8a866de7c0ccbc4a9 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 5 Apr 2022 17:05:50 +0200 Subject: [PATCH 19/90] add release note and instructions on global gas limit --- config.default.yaml | 2 +- doc/release_notes.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config.default.yaml b/config.default.yaml index 128c7bac..6983945c 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -39,7 +39,7 @@ enable: electricity: voltages: [220., 300., 380.] - gaslimit: false + gaslimit: false # global gas usage limit of X MWh_th co2limit: 7.75e+7 # 0.05 * 3.1e9*0.5 co2base: 1.487e+9 agg_p_nom_limits: data/agg_p_nom_minmax.csv diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 4ef5e956..78a9997d 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -22,6 +22,8 @@ Energy Security Release (April 2022) * old: ``estimate_renewable_capacities_from_capacity_stats`` * new: ``estimate_renewable_capacities`` +* Add function to add global constraint on use of gas in :mod:`prepare_network`. This can be activated with `electricity: gaslimit:` given in MWh. + * The powerplants that have been shut down before 2021 are filtered out. **Bugs and Compatibility** From 3678e5c523080190468e2ce235663df43e644714 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 7 Apr 2022 14:39:34 +0200 Subject: [PATCH 20/90] Add operational reserve margin constraint analogous to GenX Co-authored-by: FabianHofmann --- config.default.yaml | 5 +++ doc/release_notes.rst | 3 ++ scripts/solve_network.py | 74 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 80 insertions(+), 2 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index 6983945c..8ea82891 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -44,6 +44,11 @@ electricity: co2base: 1.487e+9 agg_p_nom_limits: data/agg_p_nom_minmax.csv + operational_reserve: # like https://genxproject.github.io/GenX/dev/core/#Reserves + epsilon_load: 0.02 # share of total load + epsilon_vres: 0.02 # share of total renewable supply + contingency: 4000 # fixed capacity in MW + extendable_carriers: Generator: [] StorageUnit: [] # battery, H2 diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 78a9997d..97a24291 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -22,6 +22,9 @@ Energy Security Release (April 2022) * old: ``estimate_renewable_capacities_from_capacity_stats`` * new: ``estimate_renewable_capacities`` +* Add operational reserve margin constraint analogous to `GenX implementation `_. + Can be activated with config setting ``electricity: operational_reserve:``. + * Add function to add global constraint on use of gas in :mod:`prepare_network`. This can be activated with `electricity: gaslimit:` given in MWh. * The powerplants that have been shut down before 2021 are filtered out. diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 4704d179..a13b1531 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -84,8 +84,9 @@ import pandas as pd import re import pypsa -from pypsa.linopf import (get_var, define_constraints, linexpr, join_exprs, - network_lopf, ilopf) +from pypsa.linopf import (get_var, define_constraints, define_variables, + linexpr, join_exprs, network_lopf, ilopf) +from pypsa.descriptors import get_switchable_as_dense as get_as_dense from pathlib import Path from vresutils.benchmark import memory_logger @@ -211,6 +212,73 @@ def add_SAFE_constraints(n, config): define_constraints(n, lhs, '>=', rhs, 'Safe', 'mintotalcap') +def add_operational_reserve_margin_constraint(n, config): + + reserve_config = config["electricity"]["operational_reserve"] + EPSILON_LOAD = reserve_config["epsilon_load"] + EPSILON_VRES = reserve_config["epsilon_vres"] + CONTINGENCY = reserve_config["contingency"] + + # Reserve Variables + reserve = get_var(n, 'Generator', 'r') + + # Share of extendable renewable capacities + ext_i = n.generators.query('p_nom_extendable').index + vres_i = n.generators_t.p_max_pu.columns + capacity_factor = n.generators_t.p_max_pu[vres_i.intersection(ext_i)] + renewable_capacity_variables = get_var(n, 'Generator', 'p_nom')[vres_i.intersection(ext_i)] + + # Left-hand-side + lhs = ( + linexpr((1, reserve)).sum(1) + + linexpr((-EPSILON_VRES * capacity_factor, renewable_capacity_variables)).sum(1) + ) + + + # Total demand at t + demand = n.loads_t.p.sum(1) + + # VRES potential of non extendable generators + capacity_factor = n.generators_t.p_max_pu[vres_i.difference(ext_i)] + renewable_capacity = n.generators.p_nom[vres_i.difference(ext_i)] + potential = (capacity_factor * renewable_capacity).sum(1) + + # Right-hand-side + rhs = EPSILON_LOAD * demand + EPSILON_VRES * potential + CONTINGENCY + + define_constraints(n, lhs, '>=', rhs, "Reserve margin") + + +def update_capacity_constraint(n): + gen_i = n.generators.index + ext_i = n.generators.query('p_nom_extendable').index + fix_i = n.generators.query('not p_nom_extendable').index + + dispatch = get_var(n, 'Generator', 'p') + reserve = get_var(n, 'Generator', 'r') + + capacity_variable = get_var(n, 'Generator', 'p_nom') + capacity_fixed = n.generators.p_nom[fix_i] + + p_max_pu = get_as_dense(n, 'Generator', 'p_max_pu') + + lhs = linexpr((1, dispatch), (1, reserve)) + lhs += linexpr((-p_max_pu[ext_i], capacity_variable)).reindex(columns=gen_i, fill_value='') + + rhs = (p_max_pu[fix_i] * capacity_fixed).reindex(columns=gen_i, fill_value=0) + + define_constraints(n, lhs, '<=', rhs, 'Generators', 'updated_capacity_constraint') + + +def add_operational_reserve_margin(n, sns, config): + + define_variables(n, 0, np.inf, 'Generator', 'r', axes=[sns, n.generators.index]) + + add_operational_reserve_margin_constraint(n, config) + + update_capacity_constraint(n) + + def add_battery_constraints(n): nodes = n.buses.index[n.buses.carrier == "battery"] if nodes.empty or ('Link', 'p_nom') not in n.variables.index: @@ -236,6 +304,8 @@ def extra_functionality(n, snapshots): add_SAFE_constraints(n, config) if 'CCL' in opts and n.generators.p_nom_extendable.any(): add_CCL_constraints(n, config) + if config["electricity"].get("operational_reserve"): + add_operational_reserve_margin(n, snapshots, config) for o in opts: if "EQ" in o: add_EQ_constraints(n, o) From 84e146834c412edb6f402255dd1fcdcc77de3cd6 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 7 Apr 2022 15:22:10 +0200 Subject: [PATCH 21/90] Apply suggestion from code review to add switch --- config.default.yaml | 1 + scripts/solve_network.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config.default.yaml b/config.default.yaml index 8ea82891..ecb9f201 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -45,6 +45,7 @@ electricity: agg_p_nom_limits: data/agg_p_nom_minmax.csv operational_reserve: # like https://genxproject.github.io/GenX/dev/core/#Reserves + activate: true epsilon_load: 0.02 # share of total load epsilon_vres: 0.02 # share of total renewable supply contingency: 4000 # fixed capacity in MW diff --git a/scripts/solve_network.py b/scripts/solve_network.py index a13b1531..0398dce0 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -304,7 +304,8 @@ def extra_functionality(n, snapshots): add_SAFE_constraints(n, config) if 'CCL' in opts and n.generators.p_nom_extendable.any(): add_CCL_constraints(n, config) - if config["electricity"].get("operational_reserve"): + reserve = config["electricity"].get("operational_reserve", {}) + if reserve.get("activate"): add_operational_reserve_margin(n, snapshots, config) for o in opts: if "EQ" in o: From 630fb9783f5acb9586f28d9732cc97ecad288524 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 7 Apr 2022 17:20:56 +0200 Subject: [PATCH 22/90] fix to operational reserve margin to work without any extendable gens --- scripts/solve_network.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 0398dce0..5d4bb780 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -221,20 +221,16 @@ def add_operational_reserve_margin_constraint(n, config): # Reserve Variables reserve = get_var(n, 'Generator', 'r') + lhs = linexpr((1, reserve)).sum(1) # Share of extendable renewable capacities ext_i = n.generators.query('p_nom_extendable').index vres_i = n.generators_t.p_max_pu.columns - capacity_factor = n.generators_t.p_max_pu[vres_i.intersection(ext_i)] - renewable_capacity_variables = get_var(n, 'Generator', 'p_nom')[vres_i.intersection(ext_i)] + if not ext_i.empty and not vres_i.empty: + capacity_factor = n.generators_t.p_max_pu[vres_i.intersection(ext_i)] + renewable_capacity_variables = get_var(n, 'Generator', 'p_nom')[vres_i.intersection(ext_i)] + lhs += linexpr((-EPSILON_VRES * capacity_factor, renewable_capacity_variables)).sum(1) - # Left-hand-side - lhs = ( - linexpr((1, reserve)).sum(1) + - linexpr((-EPSILON_VRES * capacity_factor, renewable_capacity_variables)).sum(1) - ) - - # Total demand at t demand = n.loads_t.p.sum(1) @@ -256,14 +252,16 @@ def update_capacity_constraint(n): dispatch = get_var(n, 'Generator', 'p') reserve = get_var(n, 'Generator', 'r') - - capacity_variable = get_var(n, 'Generator', 'p_nom') + capacity_fixed = n.generators.p_nom[fix_i] p_max_pu = get_as_dense(n, 'Generator', 'p_max_pu') lhs = linexpr((1, dispatch), (1, reserve)) - lhs += linexpr((-p_max_pu[ext_i], capacity_variable)).reindex(columns=gen_i, fill_value='') + + if not ext_i.empty: + capacity_variable = get_var(n, 'Generator', 'p_nom') + lhs += linexpr((-p_max_pu[ext_i], capacity_variable)).reindex(columns=gen_i, fill_value='') rhs = (p_max_pu[fix_i] * capacity_fixed).reindex(columns=gen_i, fill_value=0) From 64424ed208812ddac2e3dd6453340acceebf2c48 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 7 Apr 2022 21:13:48 +0200 Subject: [PATCH 23/90] add results dir to simplify multiple configs (like PyPSA-Eur-Sec) --- Snakefile | 70 ++++++++++++++++++++++----------------------- config.default.yaml | 2 +- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/Snakefile b/Snakefile index f3a894fc..37f0ddca 100644 --- a/Snakefile +++ b/Snakefile @@ -15,7 +15,7 @@ configfile: "config.yaml" COSTS="data/costs.csv" ATLITE_NPROCESSES = config['atlite'].get('nprocesses', 4) - +RDIR = config["results_dir"] wildcard_constraints: simpl="[a-zA-Z0-9]*|all", @@ -25,19 +25,19 @@ wildcard_constraints: rule cluster_all_networks: - input: expand("networks/elec_s{simpl}_{clusters}.nc", **config['scenario']) + input: expand(RDIR + "networks/elec_s{simpl}_{clusters}.nc", **config['scenario']) rule extra_components_all_networks: - input: expand("networks/elec_s{simpl}_{clusters}_ec.nc", **config['scenario']) + input: expand(RDIR + "networks/elec_s{simpl}_{clusters}_ec.nc", **config['scenario']) rule prepare_all_networks: - input: expand("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) + input: expand(RDIR + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) rule solve_all_networks: - input: expand("results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) + input: expand(RDIR + "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) if config['enable'].get('prepare_links_p_nom', False): @@ -228,7 +228,7 @@ rule add_electricity: nuts3_shapes='resources/nuts3_shapes.geojson', **{f"profile_{tech}": f"resources/profile_{tech}.nc" for tech in config['renewable']} - output: "networks/elec.nc" + output: RDIR + "/prenetworks/elec.nc" log: "logs/add_electricity.log" benchmark: "benchmarks/add_electricity" threads: 1 @@ -238,16 +238,16 @@ rule add_electricity: rule simplify_network: input: - network='networks/elec.nc', + network=RDIR + '/prenetworks/elec.nc', tech_costs=COSTS, - regions_onshore="resources/regions_onshore.geojson", - regions_offshore="resources/regions_offshore.geojson" + regions_onshore="/resources/regions_onshore.geojson", + regions_offshore="/resources/regions_offshore.geojson" output: - network='networks/elec_s{simpl}.nc', - regions_onshore="resources/regions_onshore_elec_s{simpl}.geojson", - regions_offshore="resources/regions_offshore_elec_s{simpl}.geojson", - busmap='resources/busmap_elec_s{simpl}.csv', - connection_costs='resources/connection_costs_s{simpl}.csv' + network=RDIR + '/prenetworks/elec_s{simpl}.nc', + regions_onshore=RDIR + "/resources/regions_onshore_elec_s{simpl}.geojson", + regions_offshore=RDIR + "/resources/regions_offshore_elec_s{simpl}.geojson", + busmap=RDIR + '/resources/busmap_elec_s{simpl}.csv', + connection_costs=RDIR + '/resources/connection_costs_s{simpl}.csv' log: "logs/simplify_network/elec_s{simpl}.log" benchmark: "benchmarks/simplify_network/elec_s{simpl}" threads: 1 @@ -257,19 +257,19 @@ rule simplify_network: rule cluster_network: input: - network='networks/elec_s{simpl}.nc', - regions_onshore="resources/regions_onshore_elec_s{simpl}.geojson", - regions_offshore="resources/regions_offshore_elec_s{simpl}.geojson", - busmap=ancient('resources/busmap_elec_s{simpl}.csv'), + network=RDIR + '/prenetworks/elec_s{simpl}.nc', + regions_onshore=RDIR + "/resources/regions_onshore_elec_s{simpl}.geojson", + regions_offshore=RDIR + "/resources/regions_offshore_elec_s{simpl}.geojson", + busmap=ancient(RDIR + '/resources/busmap_elec_s{simpl}.csv'), custom_busmap=("data/custom_busmap_elec_s{simpl}_{clusters}.csv" if config["enable"].get("custom_busmap", False) else []), tech_costs=COSTS output: - network='networks/elec_s{simpl}_{clusters}.nc', - regions_onshore="resources/regions_onshore_elec_s{simpl}_{clusters}.geojson", - regions_offshore="resources/regions_offshore_elec_s{simpl}_{clusters}.geojson", - busmap="resources/busmap_elec_s{simpl}_{clusters}.csv", - linemap="resources/linemap_elec_s{simpl}_{clusters}.csv" + network=RDIR + '/prenetworks/elec_s{simpl}_{clusters}.nc', + regions_onshore=RDIR + "/resources/regions_onshore_elec_s{simpl}_{clusters}.geojson", + regions_offshore=RDIR + "/resources/regions_offshore_elec_s{simpl}_{clusters}.geojson", + busmap=RDIR + "/resources/busmap_elec_s{simpl}_{clusters}.csv", + linemap=RDIR + "/resources/linemap_elec_s{simpl}_{clusters}.csv" log: "logs/cluster_network/elec_s{simpl}_{clusters}.log" benchmark: "benchmarks/cluster_network/elec_s{simpl}_{clusters}" threads: 1 @@ -279,9 +279,9 @@ rule cluster_network: rule add_extra_components: input: - network='networks/elec_s{simpl}_{clusters}.nc', + network=RDIR + '/prenetworks/elec_s{simpl}_{clusters}.nc', tech_costs=COSTS, - output: 'networks/elec_s{simpl}_{clusters}_ec.nc' + output: RDIR + '/prenetworks/elec_s{simpl}_{clusters}_ec.nc' log: "logs/add_extra_components/elec_s{simpl}_{clusters}.log" benchmark: "benchmarks/add_extra_components/elec_s{simpl}_{clusters}_ec" threads: 1 @@ -290,8 +290,8 @@ rule add_extra_components: rule prepare_network: - input: 'networks/elec_s{simpl}_{clusters}_ec.nc', tech_costs=COSTS - output: 'networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc' + input: RDIR + '/prenetworks/elec_s{simpl}_{clusters}_ec.nc', tech_costs=COSTS + output: RDIR + '/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc' log: "logs/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.log" benchmark: "benchmarks/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 1 @@ -320,8 +320,8 @@ def memory(w): rule solve_network: - input: "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + input: RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + output: RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" log: solver=normpath("logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"), python="logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", @@ -335,9 +335,9 @@ rule solve_network: rule solve_operations_network: input: - unprepared="networks/elec_s{simpl}_{clusters}_ec.nc", - optimized="results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" + unprepared=RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec.nc", + optimized=RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + output: RDIR + "postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" log: solver=normpath("logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"), python="logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", @@ -351,7 +351,7 @@ rule solve_operations_network: rule plot_network: input: - network="results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + network=RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", tech_costs=COSTS output: only_map="results/plots/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}.{ext}", @@ -369,7 +369,7 @@ def input_make_summary(w): else: ll = w.ll return ([COSTS] + - expand("results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + expand(RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", ll=ll, **{k: config["scenario"][k] if getattr(w, k) == "all" else getattr(w, k) for k in ["simpl", "clusters", "opts"]})) @@ -390,7 +390,7 @@ rule plot_summary: def input_plot_p_nom_max(w): - return [("networks/elec_s{simpl}{maybe_cluster}.nc" + return [(RDIR + "/postnetworks/elec_s{simpl}{maybe_cluster}.nc" .format(maybe_cluster=('' if c == 'full' else ('_' + c)), **w)) for c in w.clusts.split(",")] diff --git a/config.default.yaml b/config.default.yaml index ecb9f201..45903b80 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -9,7 +9,7 @@ logging: level: INFO format: '%(levelname)s:%(name)s:%(message)s' -summary_dir: results +results_dir: results/your-run-name scenario: simpl: [''] From b1143dc39b332020c03241b1981691156b2b447a Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 7 Apr 2022 21:48:00 +0200 Subject: [PATCH 24/90] env: update to powerplantmatching >= v0.5.2 --- envs/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 9772d882..7bcb8163 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -56,4 +56,4 @@ dependencies: - pip: - vresutils>=0.3.1 - tsam>=1.1.0 - - powerplantmatching>=0.5.1 + - powerplantmatching>=0.5.2 From f474f8bce43ac671e9b6a477c45494c9bc0c8dbd Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 7 Apr 2022 22:07:18 +0200 Subject: [PATCH 25/90] finetuning of results dir --- Snakefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Snakefile b/Snakefile index 37f0ddca..87d31cd5 100644 --- a/Snakefile +++ b/Snakefile @@ -25,19 +25,19 @@ wildcard_constraints: rule cluster_all_networks: - input: expand(RDIR + "networks/elec_s{simpl}_{clusters}.nc", **config['scenario']) + input: expand(RDIR + "/prenetworks/elec_s{simpl}_{clusters}.nc", **config['scenario']) rule extra_components_all_networks: - input: expand(RDIR + "networks/elec_s{simpl}_{clusters}_ec.nc", **config['scenario']) + input: expand(RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec.nc", **config['scenario']) rule prepare_all_networks: - input: expand(RDIR + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) + input: expand(RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) rule solve_all_networks: - input: expand(RDIR + "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) + input: expand(RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) if config['enable'].get('prepare_links_p_nom', False): @@ -240,8 +240,8 @@ rule simplify_network: input: network=RDIR + '/prenetworks/elec.nc', tech_costs=COSTS, - regions_onshore="/resources/regions_onshore.geojson", - regions_offshore="/resources/regions_offshore.geojson" + regions_onshore="resources/regions_onshore.geojson", + regions_offshore="resources/regions_offshore.geojson" output: network=RDIR + '/prenetworks/elec_s{simpl}.nc', regions_onshore=RDIR + "/resources/regions_onshore_elec_s{simpl}.geojson", @@ -337,7 +337,7 @@ rule solve_operations_network: input: unprepared=RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec.nc", optimized=RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: RDIR + "postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" + output: RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" log: solver=normpath("logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"), python="logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", From f9ede37a022975b424cfb404c4dd14da59a667e3 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Apr 2022 10:26:16 +0200 Subject: [PATCH 26/90] allow varying marginal cost of carrier in opts wc: CCGT+m2.0 --- scripts/prepare_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 27aacacf..2664d362 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -257,7 +257,7 @@ if __name__ == "__main__": if oo[0].startswith(tuple(suptechs)): carrier = oo[0] # handles only p_nom_max as stores and lines have no potentials - attr_lookup = {"p": "p_nom_max", "c": "capital_cost"} + attr_lookup = {"p": "p_nom_max", "c": "capital_cost", "m": "marginal_cost"} attr = attr_lookup[oo[1][0]] factor = float(oo[1][1:]) if carrier == "AC": # lines do not have carrier From bd75953674a6d02dc43027cc75761a22de2b662e Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Apr 2022 10:26:38 +0200 Subject: [PATCH 27/90] add logging to co2limit2 --- scripts/prepare_network.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 2664d362..52e2eff4 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -243,8 +243,10 @@ if __name__ == "__main__": if len(m) > 0: co2limit = float(m[0]) * snakemake.config['electricity']['co2base'] add_co2limit(n, co2limit, Nyears) + logger.info("Setting CO2 limit according to wildcard value.") else: add_co2limit(n, snakemake.config['electricity']['co2limit'], Nyears) + logger.info("Setting CO2 limit according to config value.") break gaslimit = snakemake.config["electricity"].get("gaslimit") From 02c06017d3f1373c7eb2107b19d5ae2cfba51278 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Apr 2022 10:27:23 +0200 Subject: [PATCH 28/90] prepare: add gas consumption limit through wildcard: CH4L --- scripts/prepare_network.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 52e2eff4..2b4e6113 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -249,9 +249,17 @@ if __name__ == "__main__": logger.info("Setting CO2 limit according to config value.") break - gaslimit = snakemake.config["electricity"].get("gaslimit") - if gaslimit: - add_gaslimit(n, gaslimit, Nyears) + for o in opts: + if "CH4L" in o: + m = re.findall("[0-9]*\.?[0-9]+$", o) + if len(m) > 0: + limit = float(m[0]) * 1e6 + add_gaslimit(n, limit, Nyears) + logger.info("Setting gas usage limit according to wildcard value.") + else: + add_gaslimit(n, snakemake.config["electricity"].get("gaslimit"), Nyears) + logger.info("Setting gas usage limit according to config value.") + break for o in opts: oo = o.split("+") From 2403650be29b60b9a902f7327f23b92697ddc723 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Apr 2022 10:27:55 +0200 Subject: [PATCH 29/90] prepare: allow varying emission prices in opts wc: e.g. Ep80.5 --- scripts/prepare_network.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 2b4e6113..178c6bb3 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -278,8 +278,16 @@ if __name__ == "__main__": sel = c.df.carrier.str.contains(carrier) c.df.loc[sel,attr] *= factor - if 'Ep' in opts: - add_emission_prices(n, snakemake.config['costs']['emission_prices']) + for o in opts: + if 'Ep' in o: + m = re.findall("[0-9]*\.?[0-9]+$", o) + if len(m) > 0: + logger.info("Setting emission prices according to wildcard value.") + add_emission_prices(n, dict(co2=float(m[0]))) + else: + logger.info("Setting emission prices according to config value.") + add_emission_prices(n, snakemake.config['costs']['emission_prices']) + break ll_type, factor = snakemake.wildcards.ll[0], snakemake.wildcards.ll[1:] set_transmission_limit(n, ll_type, factor, costs, Nyears) From e5cb2d34fbf63ee92f18780fe7d4aebcf22a4204 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 8 Apr 2022 11:30:29 +0200 Subject: [PATCH 30/90] env: update to powerplantmatching >= 0.5.3 --- envs/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 7bcb8163..9c50d8dd 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -56,4 +56,4 @@ dependencies: - pip: - vresutils>=0.3.1 - tsam>=1.1.0 - - powerplantmatching>=0.5.2 + - powerplantmatching>=0.5.3 From e37a3f57ab7fb0e187bacbcd6085b4988df06dc8 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 8 Apr 2022 12:23:24 +0200 Subject: [PATCH 31/90] adjust biomass capacities --- config.default.yaml | 2 +- scripts/build_powerplants.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/config.default.yaml b/config.default.yaml index ecb9f201..e62c6c3c 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -61,7 +61,7 @@ electricity: H2: 168 # use pandas query strings here, e.g. Country not in ['Germany'] - powerplants_filter: (DateOut >= 2021 or DateOut != DateOut) + powerplants_filter: (DateOut >= 2022 or DateOut != DateOut) # use pandas query strings here, e.g. Country in ['Germany'] custom_powerplants: false conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index 9a7c9e23..00b1a9a3 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -115,6 +115,14 @@ if __name__ == "__main__": .replace({'Technology': {'Steam Turbine': 'OCGT', "Combustion Engine": "OCGT"}}) .assign(Fueltype=replace_natural_gas_by_technology)) + # Correct bioenergy for countries where possible + opsd = pm.data.OPSD_VRE().powerplant.convert_country_to_alpha2() + opsd = opsd.query('Country in @countries and Fueltype == "Bioenergy"') + opsd['Fueltype'] = 'biomass' + available_countries = opsd.Country.unique() + ppl = ppl.query('not (Country in @available_countries and Fueltype == "Bioenergy")') + ppl = pd.concat([ppl, opsd]) + ppl_query = snakemake.config['electricity']['powerplants_filter'] if isinstance(ppl_query, str): ppl.query(ppl_query, inplace=True) From 40c882f0e968f45fc3b80beff1cfa7c24f70e9d4 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Apr 2022 13:19:25 +0200 Subject: [PATCH 32/90] solve: allow to parse load shedding cost in config --- scripts/solve_network.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 5d4bb780..a5ebfe6e 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -100,17 +100,19 @@ def prepare_network(n, solve_opts): for df in (n.generators_t.p_max_pu, n.storage_units_t.inflow): df.where(df>solve_opts['clip_p_max_pu'], other=0., inplace=True) - if solve_opts.get('load_shedding'): + load_shedding = solve_opts.get('load_shedding') + if load_shedding: n.add("Carrier", "Load") buses_i = n.buses.query("carrier == 'AC'").index + if not np.isscalar(load_shedding): load_shedding = 1e2 + # intersect between macroeconomic and surveybased + # willingness to pay + # http://journal.frontiersin.org/article/10.3389/fenrg.2015.00055/full) n.madd("Generator", buses_i, " load", bus=buses_i, carrier='load', sign=1e-3, # Adjust sign to measure p and p_nom in kW instead of MW - marginal_cost=1e2, # Eur/kWh - # intersect between macroeconomic and surveybased - # willingness to pay - # http://journal.frontiersin.org/article/10.3389/fenrg.2015.00055/full + marginal_cost=load_shedding, p_nom=1e9 # kW ) From 881c822437a972cd49bfd30503e677e0142d9e5e Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Apr 2022 13:20:43 +0200 Subject: [PATCH 33/90] Snakefile: add powerplants to results-dir --- Snakefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Snakefile b/Snakefile index 87d31cd5..b385a0db 100644 --- a/Snakefile +++ b/Snakefile @@ -221,7 +221,7 @@ rule add_electricity: base_network='networks/base.nc', tech_costs=COSTS, regions="resources/regions_onshore.geojson", - powerplants='resources/powerplants.csv', + powerplants=RDIR + '/resources/powerplants.csv', hydro_capacities='data/bundle/hydro_capacities.csv', geth_hydro_capacities='data/geth2015_hydro_capacities.csv', load='resources/load.csv', @@ -268,7 +268,7 @@ rule cluster_network: network=RDIR + '/prenetworks/elec_s{simpl}_{clusters}.nc', regions_onshore=RDIR + "/resources/regions_onshore_elec_s{simpl}_{clusters}.geojson", regions_offshore=RDIR + "/resources/regions_offshore_elec_s{simpl}_{clusters}.geojson", - busmap=RDIR + "/resources/busmap_elec_s{simpl}_{clusters}.csv", + busmap=RDIR + "/resources/busmap_elec_s{simpl}_{clusters}.csv", linemap=RDIR + "/resources/linemap_elec_s{simpl}_{clusters}.csv" log: "logs/cluster_network/elec_s{simpl}_{clusters}.log" benchmark: "benchmarks/cluster_network/elec_s{simpl}_{clusters}" From f0d0edcf542c5db84b6a02ae838fb489940b5563 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Apr 2022 13:20:43 +0200 Subject: [PATCH 34/90] Snakefile: add powerplants to results-dir --- Snakefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Snakefile b/Snakefile index 87d31cd5..df3f7cac 100644 --- a/Snakefile +++ b/Snakefile @@ -84,7 +84,7 @@ rule build_powerplants: input: base_network="networks/base.nc", custom_powerplants="data/custom_powerplants.csv" - output: "resources/powerplants.csv" + output: RDIR + "/resources/powerplants.csv" log: "logs/build_powerplants.log" threads: 1 resources: mem_mb=500 @@ -221,7 +221,7 @@ rule add_electricity: base_network='networks/base.nc', tech_costs=COSTS, regions="resources/regions_onshore.geojson", - powerplants='resources/powerplants.csv', + powerplants=RDIR + '/resources/powerplants.csv', hydro_capacities='data/bundle/hydro_capacities.csv', geth_hydro_capacities='data/geth2015_hydro_capacities.csv', load='resources/load.csv', @@ -268,7 +268,7 @@ rule cluster_network: network=RDIR + '/prenetworks/elec_s{simpl}_{clusters}.nc', regions_onshore=RDIR + "/resources/regions_onshore_elec_s{simpl}_{clusters}.geojson", regions_offshore=RDIR + "/resources/regions_offshore_elec_s{simpl}_{clusters}.geojson", - busmap=RDIR + "/resources/busmap_elec_s{simpl}_{clusters}.csv", + busmap=RDIR + "/resources/busmap_elec_s{simpl}_{clusters}.csv", linemap=RDIR + "/resources/linemap_elec_s{simpl}_{clusters}.csv" log: "logs/cluster_network/elec_s{simpl}_{clusters}.log" benchmark: "benchmarks/cluster_network/elec_s{simpl}_{clusters}" From d53487f822e717e5bbe3a1e7d93527e84df70176 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 8 Apr 2022 14:23:33 +0200 Subject: [PATCH 35/90] build_powerplants: fix duplicated names per bus --- scripts/build_powerplants.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index 00b1a9a3..cc01d373 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -118,7 +118,7 @@ if __name__ == "__main__": # Correct bioenergy for countries where possible opsd = pm.data.OPSD_VRE().powerplant.convert_country_to_alpha2() opsd = opsd.query('Country in @countries and Fueltype == "Bioenergy"') - opsd['Fueltype'] = 'biomass' + opsd['Name'] = "Biomass" available_countries = opsd.Country.unique() ppl = ppl.query('not (Country in @available_countries and Fueltype == "Bioenergy")') ppl = pd.concat([ppl, opsd]) @@ -148,4 +148,8 @@ if __name__ == "__main__": if bus_null_b.any(): logging.warning(f"Couldn't find close bus for {bus_null_b.sum()} powerplants") + # TODO: This has to fixed in PPM, some powerplants are still duplicated + cumcount = ppl.groupby(['bus', 'Fueltype']).cumcount() + 1 + ppl.Name = ppl.Name.where(cumcount == 1, ppl.Name + " " + cumcount.astype(str)) + ppl.to_csv(snakemake.output[0]) From 65790cd065f32934739a51248bcaf74ba0bdb760 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 8 Apr 2022 14:34:52 +0200 Subject: [PATCH 36/90] build_powerplants: remove non-assigned ppls --- scripts/build_powerplants.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index cc01d373..74f53d80 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -146,7 +146,9 @@ if __name__ == "__main__": bus_null_b = ppl["bus"].isnull() if bus_null_b.any(): - logging.warning(f"Couldn't find close bus for {bus_null_b.sum()} powerplants") + logging.warning(f"Couldn't find close bus for {bus_null_b.sum()} powerplants. " + "Removing them from the powerplants list.") + ppl = ppl[~bus_null_b] # TODO: This has to fixed in PPM, some powerplants are still duplicated cumcount = ppl.groupby(['bus', 'Fueltype']).cumcount() + 1 From 40425a7767e614163aae9c393e2ae6ae84129b59 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 8 Apr 2022 15:25:05 +0200 Subject: [PATCH 37/90] build powerplants: use map_country bus function for bus attachement --- scripts/build_powerplants.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index 74f53d80..c1ee4127 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -79,6 +79,7 @@ import powerplantmatching as pm import pandas as pd import numpy as np +from powerplantmatching.export import map_country_bus from scipy.spatial import cKDTree as KDTree logger = logging.getLogger(__name__) @@ -87,8 +88,7 @@ logger = logging.getLogger(__name__) def add_custom_powerplants(ppl, custom_powerplants, custom_ppl_query=False): if not custom_ppl_query: return ppl - add_ppls = pd.read_csv(custom_powerplants, index_col=0, - dtype={'bus': 'str'}) + add_ppls = pd.read_csv(custom_powerplants, index_col=0, dtype={'bus': 'str'}) if isinstance(custom_ppl_query, str): add_ppls.query(custom_ppl_query, inplace=True) return pd.concat([ppl, add_ppls], sort=False, ignore_index=True, verify_integrity=True) @@ -131,18 +131,12 @@ if __name__ == "__main__": custom_ppl_query = snakemake.config['electricity']['custom_powerplants'] ppl = add_custom_powerplants(ppl, snakemake.input.custom_powerplants, custom_ppl_query) - cntries_without_ppl = [c for c in countries if c not in ppl.Country.unique()] + countries_wo_ppl = [c for c in countries if c not in ppl.Country.unique()] + if countries_wo_ppl: + logging.warning(f"No powerplants known in: {', '.join(countries_wo_ppl)}") - for c in countries: - substation_i = n.buses.query('substation_lv and country == @c').index - kdtree = KDTree(n.buses.loc[substation_i, ['x','y']].values) - ppl_i = ppl.query('Country == @c').index - - tree_i = kdtree.query(ppl.loc[ppl_i, ['lon','lat']].values)[1] - ppl.loc[ppl_i, 'bus'] = substation_i.append(pd.Index([np.nan]))[tree_i] - - if cntries_without_ppl: - logging.warning(f"No powerplants known in: {', '.join(cntries_without_ppl)}") + substations = n.buses.query('substation_lv') + ppl = map_country_bus(ppl, substations) bus_null_b = ppl["bus"].isnull() if bus_null_b.any(): @@ -154,4 +148,4 @@ if __name__ == "__main__": cumcount = ppl.groupby(['bus', 'Fueltype']).cumcount() + 1 ppl.Name = ppl.Name.where(cumcount == 1, ppl.Name + " " + cumcount.astype(str)) - ppl.to_csv(snakemake.output[0]) + ppl.reset_index(drop=True).to_csv(snakemake.output[0]) From c412a61013feae39d3c3451a2fbd938f6c65368e Mon Sep 17 00:00:00 2001 From: euronion <42553970+euronion@users.noreply.github.com> Date: Fri, 8 Apr 2022 15:41:23 +0200 Subject: [PATCH 38/90] Add country-specific EAF restriction for NPPs. (#361) * Add country-specific EAF restriction for NPPs. Based on historic figures from IAEA. * Update release_notes.rst --- config.default.yaml | 25 ++++++++++++++++++++++++- doc/release_notes.rst | 5 ++++- scripts/add_electricity.py | 22 +++++++++++++++++++--- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index e62c6c3c..4f1708b4 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -64,7 +64,30 @@ electricity: powerplants_filter: (DateOut >= 2022 or DateOut != DateOut) # use pandas query strings here, e.g. Country in ['Germany'] custom_powerplants: false - conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] + conventional_carriers: + technologies: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] + # Limit energy availability from these sources -> p_max_pu + # syntax: + # : or : + energy_availability_factors: + # From IAEA + # https://pris.iaea.org/PRIS/WorldStatistics/ThreeYrsEnergyAvailabilityFactor.aspx (2022-04-08) + nuclear: + BE: 0.65 + BG: 0.89 + CZ: 0.82 + FI: 0.92 + FR: 0.70 + DE: 0.88 + HU: 0.90 + NL: 0.86 + RO: 0.92 + SK: 0.89 + SI: 0.94 + ES: 0.89 + SE: 0.82 + CH: 0.86 + GB: 0.67 renewable_capacities_from_OPSD: [] # onwind, offwind, solar estimate_renewable_capacities: diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 97a24291..f07176ad 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -25,7 +25,10 @@ Energy Security Release (April 2022) * Add operational reserve margin constraint analogous to `GenX implementation `_. Can be activated with config setting ``electricity: operational_reserve:``. -* Add function to add global constraint on use of gas in :mod:`prepare_network`. This can be activated with `electricity: gaslimit:` given in MWh. +* Add function to add global constraint on use of gas in :mod:`prepare_network`. This can be activated with `electricity: gaslimit:` given in MWh. + +* Add configuration option to implement Energy Availability Factors (EAFs) for conventional generation technologies. +* Implement country-specific EAFs for nuclear power plants based on IAEA 2018-2020 reported country averages. * The powerplants that have been shut down before 2021 are filtered out. diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 961b87dc..ab3aa321 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -290,8 +290,9 @@ def attach_wind_and_solar(n, costs, input_profiles, technologies, line_length_fa p_max_pu=ds['profile'].transpose('time', 'bus').to_pandas()) -def attach_conventional_generators(n, costs, ppl, carriers): +def attach_conventional_generators(n, costs, ppl, conventional_carriers): + carriers = conventional_carriers["technologies"] _add_missing_carriers_from_costs(n, costs, carriers) ppl = (ppl.query('carrier in @carriers').join(costs, on='carrier') @@ -309,6 +310,22 @@ def attach_conventional_generators(n, costs, ppl, carriers): capital_cost=0) logger.warning(f'Capital costs for conventional generators put to 0 EUR/MW.') + + for k,v in conventional_carriers["energy_availability_factors"].items(): + + # Generators with technology affected + idx = n.generators.query("carrier == @k").index + + if isinstance(v, float): + # Single value affecting all generators of technology k indiscriminantely of country + n.generators.loc[idx, "p_max_pu"] = v + elif isinstance(v, dict): + v = pd.Series(v) + + # Values affecting generators of technology k country-specific + # First map generator buses to countries; then map countries to p_max_pu + n.generators["p_max_pu"] = n.generators.loc[idx]["bus"].replace(n.buses["country"]).replace(v) + def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **config): @@ -556,8 +573,7 @@ if __name__ == "__main__": update_transmission_costs(n, costs, snakemake.config['lines']['length_factor']) - carriers = snakemake.config['electricity']['conventional_carriers'] - attach_conventional_generators(n, costs, ppl, carriers) + attach_conventional_generators(n, costs, ppl, snakemake.config["electricity"]["conventional_carriers"]) carriers = snakemake.config['renewable'] attach_wind_and_solar(n, costs, snakemake.input, carriers, snakemake.config['lines']['length_factor']) From 4712bfc893f8e32816d409a06f293a6e2f129b66 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Apr 2022 19:55:40 +0200 Subject: [PATCH 39/90] fix nuclear EAF processing code --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index ab3aa321..6c8b6de6 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -324,7 +324,7 @@ def attach_conventional_generators(n, costs, ppl, conventional_carriers): # Values affecting generators of technology k country-specific # First map generator buses to countries; then map countries to p_max_pu - n.generators["p_max_pu"] = n.generators.loc[idx]["bus"].replace(n.buses["country"]).replace(v) + n.generators.loc[idx, "p_max_pu"] = n.generators.loc[idx]["bus"].replace(n.buses["country"]).replace(v) From 732d114416d99b9402fea656611e6a44c201ea5b Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Apr 2022 19:59:27 +0200 Subject: [PATCH 40/90] add results_dir to all configs --- config.tutorial.yaml | 2 +- test/config.test1.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 225d8f78..3238329c 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -9,7 +9,7 @@ logging: level: INFO format: '%(levelname)s:%(name)s:%(message)s' -summary_dir: results +results_dir: results/your-run-name scenario: simpl: [''] diff --git a/test/config.test1.yaml b/test/config.test1.yaml index a9ce1e50..600a3aac 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -8,7 +8,7 @@ logging: level: INFO format: '%(levelname)s:%(name)s:%(message)s' -summary_dir: results +results_dir: results/your-run-name scenario: simpl: [''] From 8b855f04787b4959cfa7f22cdb1a2c3675a71155 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 7 Jun 2022 15:17:49 +0200 Subject: [PATCH 41/90] add_elecitricity: scale only missing renewable capacities from OPSD --- scripts/add_electricity.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 6c8b6de6..b2a6f7f4 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -478,18 +478,16 @@ def attach_extendable_generators(n, costs, ppl, carriers): def attach_OPSD_renewables(n, techs): - available = ['DE', 'FR', 'PL', 'CH', 'DK', 'CZ', 'SE', 'GB'] tech_map = {'Onshore': 'onwind', 'Offshore': 'offwind', 'Solar': 'solar'} - countries = set(available) & set(n.buses.country) tech_map = {k: v for k, v in tech_map.items() if v in techs} if not tech_map: return - logger.info(f'Using OPSD renewable capacities in {", ".join(countries)} ' - f'for technologies {", ".join(tech_map.values())}.') + tech_string = ", ".join(tech_map.values()) + logger.info(f'Using OPSD renewable capacities for technologies {tech_string}.') - df = pd.concat([pm.data.OPSD_VRE_country(c) for c in countries]) + df = pm.data.OPSD_VRE().powerplant.convert_country_to_alpha2() technology_b = ~df.Technology.isin(['Onshore', 'Offshore']) df['Fueltype'] = df.Fueltype.where(technology_b, df.Technology) df = df.query('Fueltype in @tech_map').powerplant.convert_country_to_alpha2() @@ -528,21 +526,27 @@ def estimate_renewable_capacities(n, config): logger.info(f"Heuristics applied to distribute renewable capacities [MW] " f"{capacities.groupby('Country').sum()}") + for ppm_technology, techs in tech_map.items(): - tech_capacities = capacities.loc[ppm_technology].reindex(countries, fill_value=0.) tech_i = n.generators.query('carrier in @techs').index - n.generators.loc[tech_i, 'p_nom'] = ( - (n.generators_t.p_max_pu[tech_i].mean() * - n.generators.loc[tech_i, 'p_nom_max']) # maximal yearly generation - .groupby(n.generators.bus.map(n.buses.country)) - .transform(lambda s: normed(s) * tech_capacities.at[s.name]) - .where(lambda s: s>0.1, 0.)) # only capacities above 100kW + stats = capacities.loc[ppm_technology].reindex(countries, fill_value=0.) + country = n.generators[tech_i].bus.map(n.buses.country) + existent = n.generators.p_nom[tech_i].groupby(country).sum() + missing = stats - existent + dist = n.generators_t.p_max_pu.mean() * n.generators.p_nom_max + + n.generators.loc[tech_i, 'p_nom'] += ( + dist[tech_i] + .groupby(country) + .transform(lambda s: normed(s) * missing[s.name]) + .where(lambda s: s>0.1, 0.) # only capacities above 100kW + ) n.generators.loc[tech_i, 'p_nom_min'] = n.generators.loc[tech_i, 'p_nom'] if expansion_limit: assert np.isscalar(expansion_limit) logger.info(f"Reducing capacity expansion limit to {expansion_limit*100:.2f}% of installed capacity.") - n.generators.loc[tech_i, 'p_nom_max'] = float(expansion_limit) * n.generators.loc[tech_i, 'p_nom_min'] + n.generators.loc[tech_i, 'p_nom_max'] = expansion_limit * n.generators.loc[tech_i, 'p_nom_min'] def add_nice_carrier_names(n, config): @@ -586,9 +590,11 @@ if __name__ == "__main__": carriers = snakemake.config['electricity']['extendable_carriers']['Generator'] attach_extendable_generators(n, costs, ppl, carriers) - estimate_renewable_capacities(n, snakemake.config) techs = snakemake.config['electricity'].get('renewable_capacities_from_OPSD', []) attach_OPSD_renewables(n, techs) + + estimate_renewable_capacities(n, snakemake.config) + update_p_nom_max(n) add_nice_carrier_names(n, snakemake.config) From eb59e68f353f4bf141cf315e834d8070691231c2 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 9 Jun 2022 20:24:28 +0200 Subject: [PATCH 42/90] Snakefile: remove RDIR selection --- Snakefile | 69 +++++++++++++++++++++++++++---------------------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/Snakefile b/Snakefile index df3f7cac..7df3a6ac 100644 --- a/Snakefile +++ b/Snakefile @@ -15,7 +15,6 @@ configfile: "config.yaml" COSTS="data/costs.csv" ATLITE_NPROCESSES = config['atlite'].get('nprocesses', 4) -RDIR = config["results_dir"] wildcard_constraints: simpl="[a-zA-Z0-9]*|all", @@ -25,19 +24,19 @@ wildcard_constraints: rule cluster_all_networks: - input: expand(RDIR + "/prenetworks/elec_s{simpl}_{clusters}.nc", **config['scenario']) + input: expand("networks/elec_s{simpl}_{clusters}.nc", **config['scenario']) rule extra_components_all_networks: - input: expand(RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec.nc", **config['scenario']) + input: expand("networks/elec_s{simpl}_{clusters}_ec.nc", **config['scenario']) rule prepare_all_networks: - input: expand(RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) + input: expand("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) rule solve_all_networks: - input: expand(RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) + input: expand("results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", **config['scenario']) if config['enable'].get('prepare_links_p_nom', False): @@ -84,7 +83,7 @@ rule build_powerplants: input: base_network="networks/base.nc", custom_powerplants="data/custom_powerplants.csv" - output: RDIR + "/resources/powerplants.csv" + output: "resources/powerplants.csv" log: "logs/build_powerplants.log" threads: 1 resources: mem_mb=500 @@ -221,14 +220,14 @@ rule add_electricity: base_network='networks/base.nc', tech_costs=COSTS, regions="resources/regions_onshore.geojson", - powerplants=RDIR + '/resources/powerplants.csv', + powerplants='resources/powerplants.csv', hydro_capacities='data/bundle/hydro_capacities.csv', geth_hydro_capacities='data/geth2015_hydro_capacities.csv', load='resources/load.csv', nuts3_shapes='resources/nuts3_shapes.geojson', **{f"profile_{tech}": f"resources/profile_{tech}.nc" for tech in config['renewable']} - output: RDIR + "/prenetworks/elec.nc" + output: "networks/elec.nc" log: "logs/add_electricity.log" benchmark: "benchmarks/add_electricity" threads: 1 @@ -238,16 +237,16 @@ rule add_electricity: rule simplify_network: input: - network=RDIR + '/prenetworks/elec.nc', + network='networks/elec.nc', tech_costs=COSTS, regions_onshore="resources/regions_onshore.geojson", regions_offshore="resources/regions_offshore.geojson" output: - network=RDIR + '/prenetworks/elec_s{simpl}.nc', - regions_onshore=RDIR + "/resources/regions_onshore_elec_s{simpl}.geojson", - regions_offshore=RDIR + "/resources/regions_offshore_elec_s{simpl}.geojson", - busmap=RDIR + '/resources/busmap_elec_s{simpl}.csv', - connection_costs=RDIR + '/resources/connection_costs_s{simpl}.csv' + network='networks/elec_s{simpl}.nc', + regions_onshore="resources/regions_onshore_elec_s{simpl}.geojson", + regions_offshore="resources/regions_offshore_elec_s{simpl}.geojson", + busmap='resources/busmap_elec_s{simpl}.csv', + connection_costs='resources/connection_costs_s{simpl}.csv' log: "logs/simplify_network/elec_s{simpl}.log" benchmark: "benchmarks/simplify_network/elec_s{simpl}" threads: 1 @@ -257,19 +256,19 @@ rule simplify_network: rule cluster_network: input: - network=RDIR + '/prenetworks/elec_s{simpl}.nc', - regions_onshore=RDIR + "/resources/regions_onshore_elec_s{simpl}.geojson", - regions_offshore=RDIR + "/resources/regions_offshore_elec_s{simpl}.geojson", - busmap=ancient(RDIR + '/resources/busmap_elec_s{simpl}.csv'), + network='networks/elec_s{simpl}.nc', + regions_onshore="resources/regions_onshore_elec_s{simpl}.geojson", + regions_offshore="resources/regions_offshore_elec_s{simpl}.geojson", + busmap=ancient('resources/busmap_elec_s{simpl}.csv'), custom_busmap=("data/custom_busmap_elec_s{simpl}_{clusters}.csv" if config["enable"].get("custom_busmap", False) else []), tech_costs=COSTS output: - network=RDIR + '/prenetworks/elec_s{simpl}_{clusters}.nc', - regions_onshore=RDIR + "/resources/regions_onshore_elec_s{simpl}_{clusters}.geojson", - regions_offshore=RDIR + "/resources/regions_offshore_elec_s{simpl}_{clusters}.geojson", - busmap=RDIR + "/resources/busmap_elec_s{simpl}_{clusters}.csv", - linemap=RDIR + "/resources/linemap_elec_s{simpl}_{clusters}.csv" + network='networks/elec_s{simpl}_{clusters}.nc', + regions_onshore="resources/regions_onshore_elec_s{simpl}_{clusters}.geojson", + regions_offshore="resources/regions_offshore_elec_s{simpl}_{clusters}.geojson", + busmap="resources/busmap_elec_s{simpl}_{clusters}.csv", + linemap="resources/linemap_elec_s{simpl}_{clusters}.csv" log: "logs/cluster_network/elec_s{simpl}_{clusters}.log" benchmark: "benchmarks/cluster_network/elec_s{simpl}_{clusters}" threads: 1 @@ -279,9 +278,9 @@ rule cluster_network: rule add_extra_components: input: - network=RDIR + '/prenetworks/elec_s{simpl}_{clusters}.nc', + network='networks/elec_s{simpl}_{clusters}.nc', tech_costs=COSTS, - output: RDIR + '/prenetworks/elec_s{simpl}_{clusters}_ec.nc' + output: 'networks/elec_s{simpl}_{clusters}_ec.nc' log: "logs/add_extra_components/elec_s{simpl}_{clusters}.log" benchmark: "benchmarks/add_extra_components/elec_s{simpl}_{clusters}_ec" threads: 1 @@ -290,8 +289,8 @@ rule add_extra_components: rule prepare_network: - input: RDIR + '/prenetworks/elec_s{simpl}_{clusters}_ec.nc', tech_costs=COSTS - output: RDIR + '/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc' + input: 'networks/elec_s{simpl}_{clusters}_ec.nc', tech_costs=COSTS + output: 'networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc' log: "logs/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.log" benchmark: "benchmarks/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 1 @@ -320,8 +319,8 @@ def memory(w): rule solve_network: - input: RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + input: "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" log: solver=normpath("logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"), python="logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", @@ -335,9 +334,9 @@ rule solve_network: rule solve_operations_network: input: - unprepared=RDIR + "/prenetworks/elec_s{simpl}_{clusters}_ec.nc", - optimized=RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" - output: RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" + unprepared="networks/elec_s{simpl}_{clusters}_ec.nc", + optimized="results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc" + output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc" log: solver=normpath("logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"), python="logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", @@ -351,7 +350,7 @@ rule solve_operations_network: rule plot_network: input: - network=RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + network="results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", tech_costs=COSTS output: only_map="results/plots/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}.{ext}", @@ -369,7 +368,7 @@ def input_make_summary(w): else: ll = w.ll return ([COSTS] + - expand(RDIR + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", + expand("results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", ll=ll, **{k: config["scenario"][k] if getattr(w, k) == "all" else getattr(w, k) for k in ["simpl", "clusters", "opts"]})) @@ -390,7 +389,7 @@ rule plot_summary: def input_plot_p_nom_max(w): - return [(RDIR + "/postnetworks/elec_s{simpl}{maybe_cluster}.nc" + return [("results/networks/elec_s{simpl}{maybe_cluster}.nc" .format(maybe_cluster=('' if c == 'full' else ('_' + c)), **w)) for c in w.clusts.split(",")] From 0ec3a8638b1c37e100e900de746f537248c3551e Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 9 Jun 2022 20:31:50 +0200 Subject: [PATCH 43/90] add_electricity & config: - refactor attachment of conventional carriers - refactor scaling of renewable carriers --- config.default.yaml | 70 ++++++++++---------- scripts/add_electricity.py | 131 +++++++++++++++++++++---------------- 2 files changed, 107 insertions(+), 94 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index edd88567..fc09dfd6 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -9,8 +9,6 @@ logging: level: INFO format: '%(levelname)s:%(name)s:%(message)s' -results_dir: results/your-run-name - scenario: simpl: [''] ll: ['copt'] @@ -50,57 +48,36 @@ electricity: epsilon_vres: 0.02 # share of total renewable supply contingency: 4000 # fixed capacity in MW - extendable_carriers: - Generator: [] - StorageUnit: [] # battery, H2 - Store: [battery, H2] - Link: [] - max_hours: battery: 6 H2: 168 + extendable_carriers: + Generator: [solar, onwind, offwind-ac, offwind-dc, OCGT] + StorageUnit: [] # battery, H2 + Store: [battery, H2] + Link: [AC, DC] + # use pandas query strings here, e.g. Country not in ['Germany'] powerplants_filter: (DateOut >= 2022 or DateOut != DateOut) # use pandas query strings here, e.g. Country in ['Germany'] custom_powerplants: false - conventional_carriers: - technologies: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] - # Limit energy availability from these sources -> p_max_pu - # syntax: - # : or : - energy_availability_factors: - # From IAEA - # https://pris.iaea.org/PRIS/WorldStatistics/ThreeYrsEnergyAvailabilityFactor.aspx (2022-04-08) - nuclear: - BE: 0.65 - BG: 0.89 - CZ: 0.82 - FI: 0.92 - FR: 0.70 - DE: 0.88 - HU: 0.90 - NL: 0.86 - RO: 0.92 - SK: 0.89 - SI: 0.94 - ES: 0.89 - SE: 0.82 - CH: 0.86 - GB: 0.67 - renewable_capacities_from_OPSD: [] # onwind, offwind, solar + + conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] + renewable_carriers: [solar, onwind, offwind-ac, offwind-dc, hydro] estimate_renewable_capacities: + enable: true + # Add capacities from OPSD data + from_opsd: true # Renewable capacities are based on existing capacities reported by IRENA - - # Reference year, any of 2000 to 2020 year: 2020 # Artificially limit maximum capacities to factor * (IRENA capacities), # i.e. 110% of 's capacities => expansion_limit: 1.1 # false: Use estimated renewable potentials determine by the workflow expansion_limit: false technology_mapping: - # Wind is the Fueltype in ppm.data.Capacity_stats, onwind, offwind-{ac,dc} the carrier in PyPSA-Eur + # Wind is the Fueltype in powerplantmatching, onwind, offwind-{ac,dc} the carrier in PyPSA-Eur Offshore: [offwind-ac, offwind-dc] Onshore: [onwind] PV: [solar] @@ -210,6 +187,27 @@ renewable: hydro_max_hours: "energy_capacity_totals_by_country" # one of energy_capacity_totals_by_country, estimate_by_large_installations or a float clip_min_inflow: 1.0 +conventional: + nuclear: + energy_availability_factors: + # From IAEA + # https://pris.iaea.org/PRIS/WorldStatistics/ThreeYrsEnergyAvailabilityFactor.aspx (2022-04-08) + BE: 0.65 + BG: 0.89 + CZ: 0.82 + FI: 0.92 + FR: 0.70 + DE: 0.88 + HU: 0.90 + NL: 0.86 + RO: 0.92 + SK: 0.89 + SI: 0.94 + ES: 0.89 + SE: 0.82 + CH: 0.86 + GB: 0.67 + lines: types: 220.: "Al/St 240/40 2-bundle 220.0" diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index b2a6f7f4..992c0b3d 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -24,8 +24,8 @@ Relevant Settings conventional_carriers: co2limit: extendable_carriers: - include_renewable_capacities_from_OPSD: - estimate_renewable_capacities_from_capacity_stats: + estimate_renewable_capacities: + load: scaling_factor: @@ -185,7 +185,7 @@ def load_powerplants(ppl_fn): 'ccgt, thermal': 'CCGT', 'hard coal': 'coal'} return (pd.read_csv(ppl_fn, index_col=0, dtype={'bus': 'str'}) .powerplant.to_pypsa_names() - .rename(columns=str.lower).drop(columns=['efficiency']) + .rename(columns=str.lower) .replace({'carrier': carrier_dict})) @@ -251,13 +251,14 @@ def update_transmission_costs(n, costs, length_factor=1.0): n.links.loc[dc_b, 'capital_cost'] = costs -def attach_wind_and_solar(n, costs, input_profiles, technologies, line_length_factor=1): +def attach_wind_and_solar(n, costs, input_profiles, technologies, extendable_carriers, line_length_factor=1): # TODO: rename tech -> carrier, technologies -> carriers - - for tech in technologies: - if tech == 'hydro': continue + _add_missing_carriers_from_costs(n, costs, technologies) + + for tech in technologies: + if tech == 'hydro': + continue - n.add("Carrier", name=tech) with xr.open_dataset(getattr(input_profiles, 'profile_' + tech)) as ds: if ds.indexes['bus'].empty: continue @@ -281,7 +282,7 @@ def attach_wind_and_solar(n, costs, input_profiles, technologies, line_length_fa n.madd("Generator", ds.indexes['bus'], ' ' + tech, bus=ds.indexes['bus'], carrier=tech, - p_nom_extendable=True, + p_nom_extendable=tech in extendable_carriers['Generator'], p_nom_max=ds['p_nom_max'].to_pandas(), weight=ds['weight'].to_pandas(), marginal_cost=costs.at[suptech, 'marginal_cost'], @@ -290,41 +291,45 @@ def attach_wind_and_solar(n, costs, input_profiles, technologies, line_length_fa p_max_pu=ds['profile'].transpose('time', 'bus').to_pandas()) -def attach_conventional_generators(n, costs, ppl, conventional_carriers): +def attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers, **config): - carriers = conventional_carriers["technologies"] + carriers = set(conventional_carriers) | set(extendable_carriers['Generator']) _add_missing_carriers_from_costs(n, costs, carriers) - ppl = (ppl.query('carrier in @carriers').join(costs, on='carrier') + ppl = (ppl.query('carrier in @carriers').join(costs, on='carrier', rsuffix='_r') .rename(index=lambda s: 'C' + str(s))) + ppl.efficiency.update(ppl.efficiency_r.dropna()) - logger.info('Adding {} generators with capacities [MW] \n{}' - .format(len(ppl), ppl.groupby('carrier').p_nom.sum())) + logger.info('Adding {} generators with capacities [GW] \n{}' + .format(len(ppl), ppl.groupby('carrier').p_nom.sum().div(1e3).round(2))) n.madd("Generator", ppl.index, carrier=ppl.carrier, bus=ppl.bus, - p_nom=ppl.p_nom, + p_nom_min=ppl.p_nom.where(ppl.carrier.isin(conventional_carriers), 0), + p_nom=ppl.p_nom.where(ppl.carrier.isin(conventional_carriers), 0), + p_nom_extendable=ppl.carrier.isin(extendable_carriers['Generator']), efficiency=ppl.efficiency, marginal_cost=ppl.marginal_cost, - capital_cost=0) - - logger.warning(f'Capital costs for conventional generators put to 0 EUR/MW.') + capital_cost=ppl.capital_cost, + build_year=ppl.datein.fillna(0).astype(int), + lifetime=(ppl.dateout - ppl.datein).fillna(9999).astype(int), + ) - for k,v in conventional_carriers["energy_availability_factors"].items(): - + for carrier in config: + # Generators with technology affected - idx = n.generators.query("carrier == @k").index + idx = n.generators.query("carrier == @carrier").index + factors = config[carrier].get("energy_availability_factors") if isinstance(v, float): # Single value affecting all generators of technology k indiscriminantely of country n.generators.loc[idx, "p_max_pu"] = v elif isinstance(v, dict): v = pd.Series(v) - # Values affecting generators of technology k country-specific # First map generator buses to countries; then map countries to p_max_pu - n.generators.loc[idx, "p_max_pu"] = n.generators.loc[idx]["bus"].replace(n.buses["country"]).replace(v) + n.generators.p_max_pu.update(n.generators.loc[idx].bus.map(v).dropna()) @@ -429,7 +434,7 @@ def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **con def attach_extendable_generators(n, costs, ppl, carriers): - + logger.warning("The function `attach_extendable_generators` is deprecated in v0.0.5.") _add_missing_carriers_from_costs(n, costs, carriers) for tech in carriers: @@ -476,24 +481,18 @@ def attach_extendable_generators(n, costs, ppl, carriers): -def attach_OPSD_renewables(n, techs): +def attach_OPSD_renewables(n, tech_map): - tech_map = {'Onshore': 'onwind', 'Offshore': 'offwind', 'Solar': 'solar'} - tech_map = {k: v for k, v in tech_map.items() if v in techs} - - if not tech_map: - return - - tech_string = ", ".join(tech_map.values()) - logger.info(f'Using OPSD renewable capacities for technologies {tech_string}.') + tech_string = ", ".join(sum(tech_map.values(), [])) + logger.info(f'Using OPSD renewable capacities for carriers {tech_string}.') df = pm.data.OPSD_VRE().powerplant.convert_country_to_alpha2() technology_b = ~df.Technology.isin(['Onshore', 'Offshore']) - df['Fueltype'] = df.Fueltype.where(technology_b, df.Technology) + df['Fueltype'] = df.Fueltype.where(technology_b, df.Technology).replace({"Solar": "PV"}) df = df.query('Fueltype in @tech_map').powerplant.convert_country_to_alpha2() - for fueltype, carrier_like in tech_map.items(): - gens = n.generators[lambda df: df.carrier.str.contains(carrier_like)] + for fueltype, carriers in tech_map.items(): + gens = n.generators[lambda df: df.carrier.isin(carriers)] buses = n.buses.loc[gens.bus.unique()] gens_per_bus = gens.groupby('bus').p_nom.count() @@ -505,32 +504,27 @@ def attach_OPSD_renewables(n, techs): n.generators.p_nom_min.update(gens.bus.map(caps).dropna()) - def estimate_renewable_capacities(n, config): - if not config["electricity"].get("estimate_renewable_capacities"): return - year = config["electricity"]["estimate_renewable_capacities"]["year"] tech_map = config["electricity"]["estimate_renewable_capacities"]["technology_mapping"] - tech_keys = list(tech_map.keys()) countries = config["countries"] expansion_limit = config["electricity"]["estimate_renewable_capacities"]["expansion_limit"] - if len(countries) == 0: return - if len(tech_map) == 0: return + if not len(countries) or not len(tech_map): return capacities = pm.data.IRENASTAT().powerplant.convert_country_to_alpha2() - capacities = capacities.query("Year == @year and Technology in @tech_keys and Country in @countries") + capacities = capacities.query("Year == @year and Technology in @tech_map and Country in @countries") capacities = capacities.groupby(["Technology", "Country"]).Capacity.sum() - logger.info(f"Heuristics applied to distribute renewable capacities [MW] " - f"{capacities.groupby('Country').sum()}") + logger.info(f"Heuristics applied to distribute renewable capacities [GW]: " + f"\n{capacities.groupby('Technology').sum().div(1e3).round(2)}") for ppm_technology, techs in tech_map.items(): tech_i = n.generators.query('carrier in @techs').index stats = capacities.loc[ppm_technology].reindex(countries, fill_value=0.) - country = n.generators[tech_i].bus.map(n.buses.country) + country = n.generators.bus[tech_i].map(n.buses.country) existent = n.generators.p_nom[tech_i].groupby(country).sum() missing = stats - existent dist = n.generators_t.p_max_pu.mean() * n.generators.p_nom_max @@ -571,29 +565,50 @@ if __name__ == "__main__": costs = load_costs(snakemake.input.tech_costs, snakemake.config['costs'], snakemake.config['electricity'], Nyears) ppl = load_powerplants(snakemake.input.powerplants) + + if "renewable_carriers" in snakemake.config['electricity']: + renewable_carriers = set(snakemake.config['renewable']) + else: + logger.warning("Key `renewable_carriers` not found in config under tag `electricity`, " + "falling back to carriers listed under `renewable`.") + renewable_carriers = snakemake.config['renewable'] + + extendable_carriers = snakemake.config['electricity']['extendable_carriers'] + if not (set(renewable_carriers) & set(extendable_carriers['Generator'])): + logger.warning(f"In future versions >= v0.0.6, extenable renewable carriers have to be " + "explicitely mentioned in `extendable_carriers`.") + + conventional_carriers = snakemake.config["electricity"]["conventional_carriers"] + attach_load(n, snakemake.input.regions, snakemake.input.load, snakemake.input.nuts3_shapes, snakemake.config['countries'], snakemake.config['load']['scaling_factor']) update_transmission_costs(n, costs, snakemake.config['lines']['length_factor']) - attach_conventional_generators(n, costs, ppl, snakemake.config["electricity"]["conventional_carriers"]) + attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers) - carriers = snakemake.config['renewable'] - attach_wind_and_solar(n, costs, snakemake.input, carriers, snakemake.config['lines']['length_factor']) + attach_wind_and_solar(n, costs, snakemake.input, renewable_carriers, extendable_carriers, snakemake.config['lines']['length_factor']) - if 'hydro' in snakemake.config['renewable']: - carriers = snakemake.config['renewable']['hydro'].pop('carriers', []) - attach_hydro(n, costs, ppl, snakemake.input.profile_hydro, snakemake.input.hydro_capacities, - carriers, **snakemake.config['renewable']['hydro']) + if 'hydro' in renewable_carriers: + conf = snakemake.config['renewable']['hydro'] + attach_hydro(n, costs, ppl, snakemake.input.profile_hydro, snakemake.input.hydro_capacities, + conf.pop('carriers', []), **conf) - carriers = snakemake.config['electricity']['extendable_carriers']['Generator'] - attach_extendable_generators(n, costs, ppl, carriers) + estimate_renewable_caps = snakemake.config['electricity'].get('estimate_renewable_capacities', {}) + if not isinstance(estimate_renewable_caps, dict): + logger.warning("The config entry `estimate_renewable_capacities` was changed to a dictionary, " + "please update your config yaml file accordingly.") + from_opsd = bool(snakemake.config["electricity"]["renewable_capacities_from_opsd"]) + estimate_renewable_caps = {"enable": True, "from_opsd": from_opsd} - techs = snakemake.config['electricity'].get('renewable_capacities_from_OPSD', []) - attach_OPSD_renewables(n, techs) + if estimate_renewable_caps["enable"]: + + if estimate_renewable_caps["from_opsd"]: + tech_map = snakemake.config["electricity"]["estimate_renewable_capacities"]["technology_mapping"] + attach_OPSD_renewables(n, tech_map) - estimate_renewable_capacities(n, snakemake.config) + estimate_renewable_capacities(n, snakemake.config) update_p_nom_max(n) From 26a56d1836fff183d5c996e76006b7a74e9f37e2 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 9 Jun 2022 20:33:13 +0200 Subject: [PATCH 44/90] update release notes --- doc/release_notes.rst | 76 ++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index f07176ad..2636ea31 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -7,13 +7,38 @@ Release Notes ########################################## -Energy Security Release (April 2022) -==================================== +Upcoming Release +================ -**New Features and Changes** +* Add an efficiency factor of 88.55% to offshore wind capacity factors + as a proxy for wake losses. More rigorous modelling is `planned `_ + [`#277 `_]. + +* The default deployment density of AC- and DC-connected offshore wind capacity is reduced from 3 MW/sqkm + to a more conservative estimate of 2 MW/sqkm [`#280 `_]. + +* Following discussion in `#285 `_ we have disabled the + correction factor for solar PV capacity factors by default while satellite data is used. + A correction factor of 0.854337 is recommended if reanalysis data like ERA5 is used. + +* Resource definitions for memory usage now follow [Snakemake standard resource definition](https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#standard-resources) ```mem_mb`` rather than ``mem``. + +* Network building is made deterministic by supplying a fixed random state to network clustering routines. + +* New network topology extracted from the ENTSO-E interactive map. + +* The unused argument ``simple_hvdc_costs`` in :mod:`add_electricity` was removed. + +* Iterative solving with impedance updates is skipped if there are no expandable lines. + +* Switch from Germany to Belgium for continuous integration and tutorial to save resources. + +* Use updated SARAH-2 and ERA5 cutouts with slightly wider scope to east and additional variables. * Added existing renewable capacities for all countries based on IRENA statistics (IRENASTAT) using new ``powerplantmatching`` version: + * The estimation is endabled by setting ``enable`` to ``True``. * Configuration of reference year for capacities can be configured (default: ``2020``) + * The list of renewables provided by the OPSD database can be used as a basis, using the tag ``from_opsd: True``. This adds the renewables from the database and fills up the missing capacities with the heuristic distribution. * Uniform expansion limit of renewable build-up based on existing capacities can be configured using ``expansion_limit`` option (default: ``false``; limited to determined renewable potentials) * Distribution of country-level capacities proportional to maximum annual energy yield for each bus region @@ -22,6 +47,8 @@ Energy Security Release (April 2022) * old: ``estimate_renewable_capacities_from_capacity_stats`` * new: ``estimate_renewable_capacities`` +* The config key ``renewable_capacities_from_OPSD`` is deprecated and was moved under the section, ``estimate_renewable_capacities``. To enable it set ``from_opsd`` to `True`. + * Add operational reserve margin constraint analogous to `GenX implementation `_. Can be activated with config setting ``electricity: operational_reserve:``. @@ -31,11 +58,22 @@ Energy Security Release (April 2022) * Implement country-specific EAFs for nuclear power plants based on IAEA 2018-2020 reported country averages. * The powerplants that have been shut down before 2021 are filtered out. - -**Bugs and Compatibility** * ``powerplantmatching>=0.5.1`` is now required for ``IRENASTATS``. +* The interpretation of ``extendable_carriers`` in the config was changed that all carriers that should be extendable have to be listed here. Before, renewable carriers were always set to be extendable. For backwards compatibility, the workflow is looking at both the listed carriers under the ``renewable`` key and the ``extendable`` key. But in the future, all of them have to be listed under ``extendable_carriers``. + +* It is now possible to set conventional power plants as extendable by adding them to the list of extendable ``Generator`` carriers in the config. + +* By having carriers in the list of ``extendable_carriers`` but not in the list of ``conventional_carriers``, the corresponding conventional power plants are set extendable without a lower capacity bound of today's capacities. + +* Now, conventional carriers have an assigned capital cost by default. + +* The ``build_year`` and ``lifetime`` column are now defined for conventional power plants. + +* A new section ``conventional`` was added to the config file. This section contains configurations for conventional carriers. Using the ``energy_availibility_factor`` key, the ``p_max_pu`` values for conventional power plants can be defined. + + Synchronisation Release - Ukraine and Moldova (17th March 2022) =============================================================== @@ -71,34 +109,6 @@ This release is not on the ``master`` branch. It can be used with git checkout synchronisation-release -Upcoming Release -================ - -* Add an efficiency factor of 88.55% to offshore wind capacity factors - as a proxy for wake losses. More rigorous modelling is `planned `_ - [`#277 `_]. - -* The default deployment density of AC- and DC-connected offshore wind capacity is reduced from 3 MW/sqkm - to a more conservative estimate of 2 MW/sqkm [`#280 `_]. - -* Following discussion in `#285 `_ we have disabled the - correction factor for solar PV capacity factors by default while satellite data is used. - A correction factor of 0.854337 is recommended if reanalysis data like ERA5 is used. - -* Resource definitions for memory usage now follow [Snakemake standard resource definition](https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#standard-resources) ```mem_mb`` rather than ``mem``. - -* Network building is made deterministic by supplying a fixed random state to network clustering routines. - -* New network topology extracted from the ENTSO-E interactive map. - -* The unused argument ``simple_hvdc_costs`` in :mod:`add_electricity` was removed. - -* Iterative solving with impedance updates is skipped if there are no expandable lines. - -* Switch from Germany to Belgium for continuous integration and tutorial to save resources. - -* Use updated SARAH-2 and ERA5 cutouts with slightly wider scope to east and additional variables. - PyPSA-Eur 0.4.0 (22th September 2021) ===================================== From f45803ff10d6e1afec101400c8e98215611515c4 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 9 Jun 2022 23:40:32 +0200 Subject: [PATCH 45/90] add_electricity: fix missing backwards compat and warnings --- scripts/add_electricity.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 24027162..880dcaad 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -25,7 +25,7 @@ Relevant Settings co2limit: extendable_carriers: estimate_renewable_capacities: - + load: scaling_factor: @@ -580,14 +580,16 @@ if __name__ == "__main__": if "renewable_carriers" in snakemake.config['electricity']: renewable_carriers = set(snakemake.config['renewable']) else: - logger.warning("Key `renewable_carriers` not found in config under tag `electricity`, " - "falling back to carriers listed under `renewable`.") + logger.warning("Missing key `renewable_carriers` under config entry `electricity`. " + "In future versions, this will raise an error. " + "Falling back to carriers listed under `renewable`.") renewable_carriers = snakemake.config['renewable'] extendable_carriers = snakemake.config['electricity']['extendable_carriers'] if not (set(renewable_carriers) & set(extendable_carriers['Generator'])): - logger.warning(f"In future versions >= v0.0.6, extenable renewable carriers have to be " - "explicitely mentioned in `extendable_carriers`.") + logger.warning("No renewables found in config entry `extendable_carriers`. " + "In future versions, these have to be explicitely listed. " + "Falling back to all renewables.") conventional_carriers = snakemake.config["electricity"]["conventional_carriers"] @@ -606,19 +608,26 @@ if __name__ == "__main__": attach_hydro(n, costs, ppl, snakemake.input.profile_hydro, snakemake.input.hydro_capacities, conf.pop('carriers', []), **conf) - estimate_renewable_caps = snakemake.config['electricity'].get('estimate_renewable_capacities', {}) - if not isinstance(estimate_renewable_caps, dict): - logger.warning("The config entry `estimate_renewable_capacities` was changed to a dictionary, " - "please update your config yaml file accordingly.") + if "estimate_renewable_capacities" not in snakemake.config['electricity']: + logger.warning("Missing key `estimate_renewable_capacities` under config entry `electricity`." + "In future versions, this will raise an error. ") + estimate_renewable_caps = {'enable': False} + if "enable" not in estimate_renewable_caps: + logger.warning("Missing key `enable` under config entry `estimate_renewable_capacities`." + "In future versions, this will raise an error. Falling back to False.") + estimate_renewable_caps = {'enable': False} + if "from_opsd" not in estimate_renewable_caps: + logger.warning("Missing key `from_opsd` under config entry `estimate_renewable_capacities`." + "In future versions, this will raise an error. " + "Falling back to whether `renewable_capacities_from_opsd` is non-empty.") from_opsd = bool(snakemake.config["electricity"]["renewable_capacities_from_opsd"]) - estimate_renewable_caps = {"enable": True, "from_opsd": from_opsd} + estimate_renewable_caps['from_opsd'] = from_opsd + - if estimate_renewable_caps["enable"]: - + if estimate_renewable_caps["enable"]: if estimate_renewable_caps["from_opsd"]: tech_map = snakemake.config["electricity"]["estimate_renewable_capacities"]["technology_mapping"] attach_OPSD_renewables(n, tech_map) - estimate_renewable_capacities(n, snakemake.config) update_p_nom_max(n) From 8cbe4e4f9dfdcbb3a5d92104e0a76497138469aa Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 10 Jun 2022 00:33:09 +0200 Subject: [PATCH 46/90] update release notes and doc --- doc/configtables/electricity.csv | 8 ++--- doc/configuration.rst | 3 -- doc/release_notes.rst | 52 +++++++++++++++++--------------- 3 files changed, 31 insertions(+), 32 deletions(-) diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index aef35350..3ec26106 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -4,7 +4,7 @@ co2limit,:math:`t_{CO_2-eq}/a`,float,Cap on total annual system carbon dioxide e co2base,:math:`t_{CO_2-eq}/a`,float,Reference value of total annual system carbon dioxide emissions if relative emission reduction target is specified in ``{opts}`` wildcard. agg_p_nom_limits,file,path,Reference to ``.csv`` file specifying per carrier generator nominal capacity constraints for individual countries if ``'CCL'`` is in ``{opts}`` wildcard. Defaults to ``data/agg_p_nom_minmax.csv``. extendable_carriers,,, --- Generator,--,"Any subset of {'OCGT','CCGT'}",Places extendable conventional power plants (OCGT and/or CCGT) where gas power plants are located today without capacity limits. +-- Generator,--,"Any extendable carrier",Defines existing or non-existing conventional and renewable power plants to be extendable during the optimization. Conventional generators can only be built/expanded where already existent today. If a listed conventional carrier is not included in the ``conventional_carriers`` list, the lower limit of the capacity expansion is set to 0. -- StorageUnit,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity. -- Store,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity. -- Link,--,Any subset of {'H2 pipeline'},Adds extendable links (H2 pipelines only) at every connection where there are lines or HVDC links without capacity limits and with zero initial capacity. Hydrogen pipelines require hydrogen storage to be modelled as ``Store``. @@ -13,7 +13,7 @@ max_hours,,, -- H2,h,float,Maximum state of charge capacity of the hydrogen storage in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_. powerplants_filter,--,"use `pandas.query `_ strings here, e.g. Country not in ['Germany']",Filter query for the default powerplant database. custom_powerplants,--,"use `pandas.query `_ strings here, e.g. Country in ['Germany']",Filter query for the custom powerplant database. -conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}",List of conventional power plants to include in the model from ``resources/powerplants.csv``. -renewable_capacities_from_OPSD,,"[solar, onwind, offwind]",List of carriers (offwind-ac and offwind-dc are included in offwind) whose capacities 'p_nom' are aligned to the `OPSD renewable power plant list `_ -estimate_renewable_capacities_from_capacitiy_stats,,, +conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}",List of conventional power plants to include in the model from ``resources/powerplants.csv``. If an included carrier is also listed in `extendable_carriers`, the capacity is taken as a lower bound. +renewable_carriers,--,"Any subset of {solar, offwind-ac, offwind-dc, hydro}",List of renewable generators to include in the model. +estimate_renewable_capacities,,, "-- Fueltype [ppm], e.g. Wind",,"list of fueltypes strings in PyPSA-Eur, e.g. [onwind, offwind-ac, offwind-dc]",converts ppm Fueltype to PyPSA-EUR Fueltype diff --git a/doc/configuration.rst b/doc/configuration.rst index a448f817..67d25228 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -91,9 +91,6 @@ Specifies the temporal range to build an energy system model for as arguments to :widths: 25,7,22,30 :file: configtables/electricity.csv -.. warning:: - Carriers in ``conventional_carriers`` must not also be in ``extendable_carriers``. - .. _atlite_cf: ``atlite`` diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 5ceea76d..1825c65c 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -36,42 +36,44 @@ Upcoming Release * Use updated SARAH-2 and ERA5 cutouts with slightly wider scope to east and additional variables. * Added existing renewable capacities for all countries based on IRENA statistics (IRENASTAT) using new ``powerplantmatching`` version: - * The estimation is endabled by setting ``enable`` to ``True``. - * Configuration of reference year for capacities can be configured (default: ``2020``) - * The list of renewables provided by the OPSD database can be used as a basis, using the tag ``from_opsd: True``. This adds the renewables from the database and fills up the missing capacities with the heuristic distribution. - * Uniform expansion limit of renewable build-up based on existing capacities can be configured using ``expansion_limit`` option - (default: ``false``; limited to determined renewable potentials) - * Distribution of country-level capacities proportional to maximum annual energy yield for each bus region - * This functionality was previously using OPSD data. - * The corresponding ``config`` entries changed, cf. ``config.default.yaml``: - * old: ``estimate_renewable_capacities_from_capacity_stats`` - * new: ``estimate_renewable_capacities`` + * The corresponding ``config`` entries changed, cf. ``config.default.yaml``: + * old: ``estimate_renewable_capacities_from_capacity_stats`` + * new: ``estimate_renewable_capacities`` + * The estimation is endabled by setting the subkey ``enable`` to ``True``. + * Configuration of reference year for capacities can be configured (default: ``2020``) + * The list of renewables provided by the OPSD database can be used as a basis, using the tag ``from_opsd: True``. This adds the renewables from the database and fills up the missing capacities with the heuristic distribution. + * Uniform expansion limit of renewable build-up based on existing capacities can be configured using ``expansion_limit`` option + (default: ``false``; limited to determined renewable potentials) + * Distribution of country-level capacities proportional to maximum annual energy yield for each bus region -* The config key ``renewable_capacities_from_OPSD`` is deprecated and was moved under the section, ``estimate_renewable_capacities``. To enable it set ``from_opsd`` to `True`. +* The config key ``renewable_capacities_from_OPSD`` is deprecated and was moved under the section, ``estimate_renewable_capacities``. To enable it, set ``from_opsd`` to `True`. -* Add operational reserve margin constraint analogous to `GenX implementation `_. - Can be activated with config setting ``electricity: operational_reserve:``. +* Add operational reserve margin constraint analogous to `GenX implementation `_. + Can be activated with config setting ``electricity: operational_reserve:``. -* Add function to add global constraint on use of gas in :mod:`prepare_network`. This can be activated with `electricity: gaslimit:` given in MWh. +* Add function to add global constraint on use of gas in :mod:`prepare_network`. This can be activated with `electricity: gaslimit:` given in MWh. -* Add configuration option to implement Energy Availability Factors (EAFs) for conventional generation technologies. -* Implement country-specific EAFs for nuclear power plants based on IAEA 2018-2020 reported country averages. +* Add configuration option to implement Energy Availability Factors (EAFs) for conventional generation technologies. -* The powerplants that have been shut down before 2021 are filtered out. - -* ``powerplantmatching>=0.5.1`` is now required for ``IRENASTATS``. +* A new section ``conventional`` was added to the config file. This section contains configurations for conventional carriers. -* The interpretation of ``extendable_carriers`` in the config was changed that all carriers that should be extendable have to be listed here. Before, renewable carriers were always set to be extendable. For backwards compatibility, the workflow is looking at both the listed carriers under the ``renewable`` key and the ``extendable`` key. But in the future, all of them have to be listed under ``extendable_carriers``. +* Implement country-specific EAFs for nuclear power plants based on IAEA 2018-2020 reported country averages. These are specified under the ``energy_availibility_factor`` key in the config entry ``conventional`` and specify the static `p_max_pu` values. -* It is now possible to set conventional power plants as extendable by adding them to the list of extendable ``Generator`` carriers in the config. +* The powerplants that have been shut down before 2021 are filtered out. -* By having carriers in the list of ``extendable_carriers`` but not in the list of ``conventional_carriers``, the corresponding conventional power plants are set extendable without a lower capacity bound of today's capacities. +* ``powerplantmatching>=0.5.1`` is now required for ``IRENASTATS``. -* Now, conventional carriers have an assigned capital cost by default. +* The inclusion of renewable carriers is now specified in the config entry ``renewable_carriers``. Before this was done by commenting/uncommenting sub-sections in the `renewable` config section. -* The ``build_year`` and ``lifetime`` column are now defined for conventional power plants. +* Now, all carriers that should be extendable have to be listed in the config entry ``extendable_carriers``. Before, renewable carriers were always set to be extendable. For backwards compatibility, the workflow is still looking at the listed carriers under the ``renewable`` key. In the future, all of them have to be listed under ``extendable_carriers``. -* A new section ``conventional`` was added to the config file. This section contains configurations for conventional carriers. Using the ``energy_availibility_factor`` key, the ``p_max_pu`` values for conventional power plants can be defined. +* It is now possible to set conventional power plants as extendable by adding them to the list of extendable ``Generator`` carriers in the config. + +* Listing conventional carriers in ``extendable_carriers`` but not in ``conventional_carriers``, sets the corresponding conventional power plants as extendable without a lower capacity bound of today's capacities. + +* Now, conventional carriers have an assigned capital cost by default. + +* The ``build_year`` and ``lifetime`` column are now defined for conventional power plants. * Fix crs bug. Change crs 4236 to 4326. From c68aa028ffff8d83a4ea3fa0dcefcf8e256aa58a Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 10 Jun 2022 00:36:07 +0200 Subject: [PATCH 47/90] add_electricity: fix missing config key --- scripts/add_electricity.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 880dcaad..bd17dbe5 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -609,18 +609,22 @@ if __name__ == "__main__": conf.pop('carriers', []), **conf) if "estimate_renewable_capacities" not in snakemake.config['electricity']: - logger.warning("Missing key `estimate_renewable_capacities` under config entry `electricity`." - "In future versions, this will raise an error. ") - estimate_renewable_caps = {'enable': False} + logger.warning("Missing key `estimate_renewable_capacities` under config entry `electricity`. " + "In future versions, this will raise an error. " + "Falling back to whether ``estimate_renewable_capacities_from_capacity_stats`` is in the config.") + if "estimate_renewable_capacities_from_capacity_stats" in snakemake.config['electricity']: + estimate_renewable_caps = {'enable': True, **snakemake.config['electricity']["estimate_renewable_capacities_from_capacity_stats"]} + else: + estimate_renewable_caps = {'enable': False} if "enable" not in estimate_renewable_caps: - logger.warning("Missing key `enable` under config entry `estimate_renewable_capacities`." + logger.warning("Missing key `enable` under config entry `estimate_renewable_capacities`. " "In future versions, this will raise an error. Falling back to False.") estimate_renewable_caps = {'enable': False} if "from_opsd" not in estimate_renewable_caps: - logger.warning("Missing key `from_opsd` under config entry `estimate_renewable_capacities`." + logger.warning("Missing key `from_opsd` under config entry `estimate_renewable_capacities`. " "In future versions, this will raise an error. " "Falling back to whether `renewable_capacities_from_opsd` is non-empty.") - from_opsd = bool(snakemake.config["electricity"]["renewable_capacities_from_opsd"]) + from_opsd = bool(snakemake.config["electricity"].get("renewable_capacities_from_opsd", False)) estimate_renewable_caps['from_opsd'] = from_opsd From 13992125bdaf3d8b7190b6c396f7ccaa2a3a75a7 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 10 Jun 2022 00:54:54 +0200 Subject: [PATCH 48/90] cluster_network: adjust generator strategy for new columns --- scripts/add_electricity.py | 2 +- scripts/cluster_network.py | 6 ++++-- scripts/simplify_network.py | 15 +++++++++++++-- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index bd17dbe5..87516a83 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -309,7 +309,7 @@ def attach_conventional_generators(n, costs, ppl, conventional_carriers, extenda ppl = (ppl.query('carrier in @carriers').join(costs, on='carrier', rsuffix='_r') .rename(index=lambda s: 'C' + str(s))) - ppl.efficiency.update(ppl.efficiency_r.dropna()) + ppl["efficiency"] = ppl.efficiency.fillna(ppl.efficiency_r) logger.info('Adding {} generators with capacities [GW] \n{}' .format(len(ppl), ppl.groupby('carrier').p_nom.sum().div(1e3).round(2))) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 642db4da..a3ce5c52 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -281,7 +281,9 @@ def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carr aggregate_generators_carriers=aggregate_carriers, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=line_length_factor, - generator_strategies={'p_nom_max': p_nom_max_strategy, 'p_nom_min': pd.Series.sum}, + generator_strategies={'p_nom_max': p_nom_max_strategy, 'p_nom_min': pd.Series.sum, + 'build_year': lambda x: 0, 'lifetime': lambda x: np.inf, + 'efficiency': np.mean}, scale_link_capital_costs=False) if not n.links.empty: @@ -342,7 +344,7 @@ if __name__ == "__main__": if snakemake.wildcards.clusters.endswith('m'): n_clusters = int(snakemake.wildcards.clusters[:-1]) - aggregate_carriers = pd.Index(n.generators.carrier.unique()).difference(renewable_carriers) + aggregate_carriers = None elif snakemake.wildcards.clusters == 'all': n_clusters = len(n.buses) aggregate_carriers = None # All diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 287dfe32..5f9aec6c 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -200,7 +200,14 @@ def _aggregate_and_move_components(n, busmap, connection_costs_to_bus, output, a _adjust_capital_costs_using_connection_costs(n, connection_costs_to_bus, output) - generators, generators_pnl = aggregategenerators(n, busmap, custom_strategies={'p_nom_min': np.sum}) + strategies = { + 'p_nom_min': np.sum, + 'p_nom_max': 'sum', + 'build_year': lambda x: 0, + 'lifetime': lambda x: np.inf, + 'efficiency': np.mean + } + generators, generators_pnl = aggregategenerators(n, busmap, custom_strategies=strategies) replace_components(n, "Generator", generators, generators_pnl) for one_port in aggregate_one_ports: @@ -351,7 +358,11 @@ def aggregate_to_substations(n, buses_i=None): aggregate_generators_carriers=None, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=1.0, - generator_strategies={'p_nom_max': 'sum'}, + generator_strategies={'p_nom_max': 'sum', + 'build_year': lambda x: 0, + 'lifetime': lambda x: np.inf, + 'efficiency': np.mean + }, scale_link_capital_costs=False) return clustering.network, busmap From 1c0975181e17b324b35e5b55986039502d0e73fc Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 10 Jun 2022 11:23:35 +0200 Subject: [PATCH 49/90] add_electricity: fix missing config key --- scripts/add_electricity.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 87516a83..ba1bf9bf 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -616,6 +616,8 @@ if __name__ == "__main__": estimate_renewable_caps = {'enable': True, **snakemake.config['electricity']["estimate_renewable_capacities_from_capacity_stats"]} else: estimate_renewable_caps = {'enable': False} + else: + estimate_renewable_caps = snakemake.config['electricity']["estimate_renewable_capacities"] if "enable" not in estimate_renewable_caps: logger.warning("Missing key `enable` under config entry `estimate_renewable_capacities`. " "In future versions, this will raise an error. Falling back to False.") From bef4967e84567434aa836d6d572c0316abe6f96f Mon Sep 17 00:00:00 2001 From: martacki Date: Mon, 20 Jun 2022 18:58:23 +0200 Subject: [PATCH 50/90] clustering strategies moved to configurables --- config.default.yaml | 4 ++++ config.tutorial.yaml | 4 ++++ scripts/cluster_network.py | 43 +++++++++++++++-------------------- scripts/simplify_network.py | 45 ++++++++++++++++++++----------------- 4 files changed, 51 insertions(+), 45 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index d2bf6159..9462719b 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -22,6 +22,10 @@ countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'F clustering: simplify: to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + aggregation_strategies: + generators: + p_nom_max: "sum" # use "min" for more conservative assumptions + p_nom_min: "sum" snapshots: start: "2013-01-01" diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 225d8f78..4dc7a94d 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -22,6 +22,10 @@ countries: ['BE'] clustering: simplify: to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + aggregation_strategies: + generators: + p_nom_max: "sum" # use "min" for more conservative assumptions + p_nom_min: "sum" snapshots: start: "2013-03-01" diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 1d5608e2..0a3d768a 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -11,11 +11,10 @@ Relevant Settings .. code:: yaml - focus_weights: + clustering: + aggregation_strategies: - renewable: (keys) - {technology}: - potential: + focus_weights: solving: solver: @@ -259,15 +258,16 @@ def busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights=None, algori def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carriers=None, - line_length_factor=1.25, potential_mode='simple', solver_name="cbc", + line_length_factor=1.25, aggregation_strategies=dict(), solver_name="cbc", algorithm="kmeans", extended_link_costs=0, focus_weights=None): - if potential_mode == 'simple': - p_nom_max_strategy = pd.Series.sum - elif potential_mode == 'conservative': - p_nom_max_strategy = pd.Series.min - else: - raise AttributeError(f"potential_mode should be one of 'simple' or 'conservative' but is '{potential_mode}'") + bus_strategies = dict(country=_make_consense("Bus", "country")) + bus_strategies.update(aggregation_strategies.get("buses", {})) + generator_strategies = aggregation_strategies.get("generators", {"p_nom_max": "sum"}) + + # this snippet supports compatibility of PyPSA and PyPSA-EUR: + if "p_nom_max" in generator_strategies: + if generator_strategies["p_nom_max"] == "min": generator_strategies["p_nom_max"] = np.min if not isinstance(custom_busmap, pd.Series): busmap = busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights, algorithm) @@ -276,19 +276,12 @@ def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carr clustering = get_clustering_from_busmap( n, busmap, - bus_strategies=dict(country=_make_consense("Bus", "country")), + bus_strategies=bus_strategies, aggregate_generators_weighted=True, aggregate_generators_carriers=aggregate_carriers, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=line_length_factor, - generator_strategies={'p_nom_max': p_nom_max_strategy, - 'p_nom_min': pd.Series.sum, - 'p_min_pu': pd.Series.mean, - 'marginal_cost': pd.Series.mean, - 'committable': np.any, - 'ramp_limit_up': pd.Series.max, - 'ramp_limit_down': pd.Series.max, - }, + generator_strategies=generator_strategies, scale_link_capital_costs=False) if not n.links.empty: @@ -375,8 +368,8 @@ if __name__ == "__main__": "The `potential` configuration option must agree for all renewable carriers, for now!" ) return v - potential_mode = consense(pd.Series([snakemake.config['renewable'][tech]['potential'] - for tech in renewable_carriers])) + aggregation_strategies = snakemake.config["clustering"].get("aggregation_strategies", {}) + custom_busmap = snakemake.config["enable"].get("custom_busmap", False) if custom_busmap: custom_busmap = pd.read_csv(snakemake.input.custom_busmap, index_col=0, squeeze=True) @@ -384,12 +377,12 @@ if __name__ == "__main__": logger.info(f"Imported custom busmap from {snakemake.input.custom_busmap}") clustering = clustering_for_n_clusters(n, n_clusters, custom_busmap, aggregate_carriers, - line_length_factor, potential_mode, + line_length_factor, aggregation_strategies, snakemake.config['solving']['solver']['name'], "kmeans", hvac_overhead_cost, focus_weights) - - update_p_nom_max(n) + update_p_nom_max(clustering.network) + clustering.network.export_to_netcdf(snakemake.output.network) for attr in ('busmap', 'linemap'): #also available: linemap_positive, linemap_negative getattr(clustering, attr).to_csv(snakemake.output[attr]) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 287dfe32..7d51c511 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -13,6 +13,10 @@ Relevant Settings .. code:: yaml + clustering: + simplify: + aggregation_strategies: + costs: USD2013_to_EUR2013: discountrate: @@ -22,10 +26,6 @@ Relevant Settings electricity: max_hours: - renewables: (keys) - {technology}: - potential: - lines: length_factor: @@ -320,7 +320,7 @@ def remove_stubs(n, costs, config, output): return n, busmap -def aggregate_to_substations(n, buses_i=None): +def aggregate_to_substations(n, config, aggregation_strategies=dict(), buses_i=None): # can be used to aggregate a selection of buses to electrically closest neighbors # if no buses are given, nodes that are no substations or without offshore connection are aggregated @@ -345,19 +345,29 @@ def aggregate_to_substations(n, buses_i=None): busmap = n.buses.index.to_series() busmap.loc[buses_i] = dist.idxmin(1) + # default aggregation strategies must be specified within the function, otherwise (when defaults are passed in + # the function's definition) they get lost in case custom values for different variables are specified in the config + bus_strategies = dict(country=_make_consense("Bus", "country")) + bus_strategies.update(aggregation_strategies.get("buses", {})) + generator_strategies = aggregation_strategies.get("generators", {"p_nom_max": "sum"}) + + # this snippet supports compatibility of PyPSA and PyPSA-EUR: + if "p_nom_max" in generator_strategies: + if generator_strategies["p_nom_max"] == "min": generator_strategies["p_nom_max"] = np.min + clustering = get_clustering_from_busmap(n, busmap, - bus_strategies=dict(country=_make_consense("Bus", "country")), + bus_strategies=bus_strategies, aggregate_generators_weighted=True, aggregate_generators_carriers=None, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=1.0, - generator_strategies={'p_nom_max': 'sum'}, + generator_strategies=generator_strategies, scale_link_capital_costs=False) return clustering.network, busmap -def cluster(n, n_clusters, config): +def cluster(n, n_clusters, config, aggregation_strategies=dict()): logger.info(f"Clustering to {n_clusters} buses") focus_weights = config.get('focus_weights', None) @@ -365,16 +375,9 @@ def cluster(n, n_clusters, config): renewable_carriers = pd.Index([tech for tech in n.generators.carrier.unique() if tech.split('-', 2)[0] in config['renewable']]) - def consense(x): - v = x.iat[0] - assert ((x == v).all() or x.isnull().all()), ( - "The `potential` configuration option must agree for all renewable carriers, for now!" - ) - return v - potential_mode = (consense(pd.Series([config['renewable'][tech]['potential'] - for tech in renewable_carriers])) - if len(renewable_carriers) > 0 else 'conservative') - clustering = clustering_for_n_clusters(n, n_clusters, custom_busmap=False, potential_mode=potential_mode, + + clustering = clustering_for_n_clusters(n, n_clusters, custom_busmap=False, + aggregation_strategies=aggregation_strategies, solver_name=config['solving']['solver']['name'], focus_weights=focus_weights) @@ -389,6 +392,8 @@ if __name__ == "__main__": n = pypsa.Network(snakemake.input.network) + aggregation_strategies = snakemake.config["clustering"].get("aggregation_strategies", {}) + n, trafo_map = simplify_network_to_380(n) Nyears = n.snapshot_weightings.objective.sum() / 8760 @@ -402,11 +407,11 @@ if __name__ == "__main__": busmaps = [trafo_map, simplify_links_map, stub_map] if snakemake.config.get('clustering', {}).get('simplify', {}).get('to_substations', False): - n, substation_map = aggregate_to_substations(n) + n, substation_map = aggregate_to_substations(n, snakemake.config, aggregation_strategies) busmaps.append(substation_map) if snakemake.wildcards.simpl: - n, cluster_map = cluster(n, int(snakemake.wildcards.simpl), snakemake.config) + n, cluster_map = cluster(n, int(snakemake.wildcards.simpl), snakemake.config, aggregation_strategies) busmaps.append(cluster_map) # some entries in n.buses are not updated in previous functions, therefore can be wrong. as they are not needed From b4d8dd8ecb827c3ed450375e80fcf3717cc2aaab Mon Sep 17 00:00:00 2001 From: martacki Date: Mon, 20 Jun 2022 19:21:08 +0200 Subject: [PATCH 51/90] add changes from PR #379 --- config.default.yaml | 9 +++++++-- config.tutorial.yaml | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index 9462719b..7162d449 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -24,8 +24,13 @@ clustering: to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) aggregation_strategies: generators: - p_nom_max: "sum" # use "min" for more conservative assumptions - p_nom_min: "sum" + p_nom_max: sum # use "min" for more conservative assumptions + p_nom_min: sum + p_min_pu: mean + marginal_cost: mean + committable: any + ramp_limit_up: max + ramp_limit_down: max snapshots: start: "2013-01-01" diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 4dc7a94d..f18f23f4 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -24,8 +24,13 @@ clustering: to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) aggregation_strategies: generators: - p_nom_max: "sum" # use "min" for more conservative assumptions - p_nom_min: "sum" + p_nom_max: sum # use "min" for more conservative assumptions + p_nom_min: sum + p_min_pu: mean + marginal_cost: mean + committable: any + ramp_limit_up: max + ramp_limit_down: max snapshots: start: "2013-03-01" From bdd0cc3aa125fa5a721357ebfc47418b79475716 Mon Sep 17 00:00:00 2001 From: martacki Date: Tue, 21 Jun 2022 18:42:49 +0200 Subject: [PATCH 52/90] clustering strats to configurables: review suggestions --- scripts/cluster_network.py | 11 +++++++---- scripts/simplify_network.py | 19 +++++++++++-------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 0a3d768a..fd66b043 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -263,11 +263,8 @@ def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carr bus_strategies = dict(country=_make_consense("Bus", "country")) bus_strategies.update(aggregation_strategies.get("buses", {})) - generator_strategies = aggregation_strategies.get("generators", {"p_nom_max": "sum"}) - # this snippet supports compatibility of PyPSA and PyPSA-EUR: - if "p_nom_max" in generator_strategies: - if generator_strategies["p_nom_max"] == "min": generator_strategies["p_nom_max"] = np.min + generator_strategies = aggregation_strategies.get("generators", {"p_nom_max": "sum"}) if not isinstance(custom_busmap, pd.Series): busmap = busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights, algorithm) @@ -369,6 +366,12 @@ if __name__ == "__main__": ) return v aggregation_strategies = snakemake.config["clustering"].get("aggregation_strategies", {}) + aggregation_strategies = {} + # translate str entries of aggregation_strategies to pd.Series functions: + aggregation_strategies = { + p: {k: getattr(pd.Series, v) for k,v in aggregation_strategies[p].items()} + for p in aggregation_strategies.keys() + } custom_busmap = snakemake.config["enable"].get("custom_busmap", False) if custom_busmap: diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 7d51c511..52e0c815 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -320,7 +320,7 @@ def remove_stubs(n, costs, config, output): return n, busmap -def aggregate_to_substations(n, config, aggregation_strategies=dict(), buses_i=None): +def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None): # can be used to aggregate a selection of buses to electrically closest neighbors # if no buses are given, nodes that are no substations or without offshore connection are aggregated @@ -345,15 +345,13 @@ def aggregate_to_substations(n, config, aggregation_strategies=dict(), buses_i=N busmap = n.buses.index.to_series() busmap.loc[buses_i] = dist.idxmin(1) - # default aggregation strategies must be specified within the function, otherwise (when defaults are passed in - # the function's definition) they get lost in case custom values for different variables are specified in the config + # default aggregation strategies must be specified within the function, otherwise (when defaults + # are passed in the function's definition) they get lost in case custom values for different + # variables are specified in the config. bus_strategies = dict(country=_make_consense("Bus", "country")) bus_strategies.update(aggregation_strategies.get("buses", {})) - generator_strategies = aggregation_strategies.get("generators", {"p_nom_max": "sum"}) - # this snippet supports compatibility of PyPSA and PyPSA-EUR: - if "p_nom_max" in generator_strategies: - if generator_strategies["p_nom_max"] == "min": generator_strategies["p_nom_max"] = np.min + generator_strategies = aggregation_strategies.get("generators", {"p_nom_max": "sum"}) clustering = get_clustering_from_busmap(n, busmap, bus_strategies=bus_strategies, @@ -393,6 +391,11 @@ if __name__ == "__main__": n = pypsa.Network(snakemake.input.network) aggregation_strategies = snakemake.config["clustering"].get("aggregation_strategies", {}) + # translate str entries of aggregation_strategies to pd.Series functions: + aggregation_strategies = { + p: {k: getattr(pd.Series, v) for k,v in aggregation_strategies[p].items()} + for p in aggregation_strategies.keys() + } n, trafo_map = simplify_network_to_380(n) @@ -407,7 +410,7 @@ if __name__ == "__main__": busmaps = [trafo_map, simplify_links_map, stub_map] if snakemake.config.get('clustering', {}).get('simplify', {}).get('to_substations', False): - n, substation_map = aggregate_to_substations(n, snakemake.config, aggregation_strategies) + n, substation_map = aggregate_to_substations(n, aggregation_strategies) busmaps.append(substation_map) if snakemake.wildcards.simpl: From 8dba48c4125a5578a65f4cf96c94f33780fb36ca Mon Sep 17 00:00:00 2001 From: martacki Date: Tue, 21 Jun 2022 19:08:22 +0200 Subject: [PATCH 53/90] clustering strats to configurables: documentation and testing --- doc/configtables/clustering.csv | 5 +++++ doc/release_notes.rst | 1 + test/config.test1.yaml | 9 +++++++++ 3 files changed, 15 insertions(+) diff --git a/doc/configtables/clustering.csv b/doc/configtables/clustering.csv index 2f63f955..f178ff5c 100644 --- a/doc/configtables/clustering.csv +++ b/doc/configtables/clustering.csv @@ -1,3 +1,8 @@ ,Unit,Values,Description simplify,,, -- to_substations,bool,"{'true','false'}","Aggregates all nodes without power injection (positive or negative, i.e. demand or generation) to electrically closest ones" +-- aggregation_strategies,,, +-- -- generators,,, +-- -- -- {key},str,"{key} can be any of the component of the generator (str). It’s value can be any that can be converted to pandas.Series using getattr(). For example one of {min, max, sum}.","Aggregates the component according to the given strategy. For example, if sum, then all values within each cluster are summed to represent the new generator." +-- -- buses,,, +-- -- -- {key},str,"{key} can be any of the component of the bus (str). It’s value can be any that can be converted to pandas.Series using getattr(). For example one of {min, max, sum}.","Aggregates the component according to the given strategy. For example, if sum, then all values within each cluster are summed to represent the new bus." diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 963a1175..c000a046 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -74,6 +74,7 @@ Upcoming Release * Update rasterio version to correctly calculate exclusion raster +* Clustering strategies for generators and buses have moved from distinct scripts to configurables to unify the process and make it more transparent. PyPSA-Eur 0.4.0 (22th September 2021) ===================================== diff --git a/test/config.test1.yaml b/test/config.test1.yaml index a9ce1e50..e3f39ab5 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -21,6 +21,15 @@ countries: ['BE'] clustering: simplify: to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + aggregation_strategies: + generators: + p_nom_max: sum # use "min" for more conservative assumptions + p_nom_min: sum + p_min_pu: mean + marginal_cost: mean + committable: any + ramp_limit_up: max + ramp_limit_down: max snapshots: start: "2013-03-01" From f6b7317d043b8d53ba779727da273bd1afd5d81b Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Thu, 23 Jun 2022 15:14:38 +0200 Subject: [PATCH 54/90] Update config.tutorial.yaml Co-authored-by: Martha Frysztacki --- config.tutorial.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 3238329c..568119cd 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -9,7 +9,6 @@ logging: level: INFO format: '%(levelname)s:%(name)s:%(message)s' -results_dir: results/your-run-name scenario: simpl: [''] From db78f9cd2adec74c75ef8c286b0cf2699a823997 Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Thu, 23 Jun 2022 15:54:22 +0200 Subject: [PATCH 55/90] Update scripts/add_electricity.py Co-authored-by: Martha Frysztacki --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index ba1bf9bf..4e4fdecd 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -599,7 +599,7 @@ if __name__ == "__main__": update_transmission_costs(n, costs, snakemake.config['lines']['length_factor']) - attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers) + attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers, **snakemake.config["conventional"]) attach_wind_and_solar(n, costs, snakemake.input, renewable_carriers, extendable_carriers, snakemake.config['lines']['length_factor']) From 441d7d56f9f25b0b187008931c3c615b1d599942 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 23 Jun 2022 16:04:49 +0200 Subject: [PATCH 56/90] fix eafs and conventional setttings --- config.default.yaml | 23 +++-------------------- scripts/add_electricity.py | 17 +++++++++-------- scripts/solve_network.py | 4 ++++ 3 files changed, 16 insertions(+), 28 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index fc09dfd6..30f1d2b6 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -43,14 +43,14 @@ electricity: agg_p_nom_limits: data/agg_p_nom_minmax.csv operational_reserve: # like https://genxproject.github.io/GenX/dev/core/#Reserves - activate: true + activate: false epsilon_load: 0.02 # share of total load epsilon_vres: 0.02 # share of total renewable supply contingency: 4000 # fixed capacity in MW max_hours: battery: 6 - H2: 168 + H2: 168 extendable_carriers: Generator: [solar, onwind, offwind-ac, offwind-dc, OCGT] @@ -189,24 +189,7 @@ renewable: conventional: nuclear: - energy_availability_factors: - # From IAEA - # https://pris.iaea.org/PRIS/WorldStatistics/ThreeYrsEnergyAvailabilityFactor.aspx (2022-04-08) - BE: 0.65 - BG: 0.89 - CZ: 0.82 - FI: 0.92 - FR: 0.70 - DE: 0.88 - HU: 0.90 - NL: 0.86 - RO: 0.92 - SK: 0.89 - SI: 0.94 - ES: 0.89 - SE: 0.82 - CH: 0.86 - GB: 0.67 + energy_availability_factors: "data/nuclear-eafs.csv" # float of file name lines: types: diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 4e4fdecd..c37a791d 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -302,7 +302,7 @@ def attach_wind_and_solar(n, costs, input_profiles, technologies, extendable_car p_max_pu=ds['profile'].transpose('time', 'bus').to_pandas()) -def attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers, **config): +def attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers, conventional_config): carriers = set(conventional_carriers) | set(extendable_carriers['Generator']) _add_missing_carriers_from_costs(n, costs, carriers) @@ -327,20 +327,21 @@ def attach_conventional_generators(n, costs, ppl, conventional_carriers, extenda lifetime=(ppl.dateout - ppl.datein).fillna(9999).astype(int), ) - for carrier in config: + for carrier in conventional_config: # Generators with technology affected idx = n.generators.query("carrier == @carrier").index - factors = config[carrier].get("energy_availability_factors") + factors = conventional_config[carrier].get("energy_availability_factors") - if isinstance(v, float): + if isinstance(factors, float): # Single value affecting all generators of technology k indiscriminantely of country - n.generators.loc[idx, "p_max_pu"] = v - elif isinstance(v, dict): - v = pd.Series(v) + n.generators.loc[idx, "p_max_pu"] = factors + elif isinstance(factors, str): + factors = pd.read_file(factors, index_col=0) # Values affecting generators of technology k country-specific # First map generator buses to countries; then map countries to p_max_pu - n.generators.p_max_pu.update(n.generators.loc[idx].bus.map(v).dropna()) + bus_factors = n.buses.country.map(factors) + n.generators.p_max_pu.update(n.generators.loc[idx].bus.map(bus_factors).dropna()) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index a5ebfe6e..06296723 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -271,6 +271,10 @@ def update_capacity_constraint(n): def add_operational_reserve_margin(n, sns, config): + """ + Build reserve margin constraints based on the formulation given in + https://genxproject.github.io/GenX/dev/core/#Reserves. + """ define_variables(n, 0, np.inf, 'Generator', 'r', axes=[sns, n.generators.index]) From 95e8a9534a3410fc75bd76bc7d8b47b97e12fe1e Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 23 Jun 2022 16:07:33 +0200 Subject: [PATCH 57/90] env: remove duplicated ppm dependency --- envs/environment.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 0c636edc..73c14fa5 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -57,4 +57,3 @@ dependencies: - pip: - vresutils>=0.3.1 - tsam>=1.1.0 - - powerplantmatching>=0.5.3 From d6930b878afab2084df172269a6778f288fdc2cc Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Thu, 23 Jun 2022 16:14:20 +0200 Subject: [PATCH 58/90] Update scripts/cluster_network.py Co-authored-by: Martha Frysztacki --- scripts/cluster_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 5166f37a..833614ab 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -352,7 +352,7 @@ if __name__ == "__main__": if snakemake.wildcards.clusters.endswith('m'): n_clusters = int(snakemake.wildcards.clusters[:-1]) - aggregate_carriers = None + aggregate_carriers = snakemake.config["electricity"].get("conventional_carriers") elif snakemake.wildcards.clusters == 'all': n_clusters = len(n.buses) aggregate_carriers = None # All From 348b14b052fcd5744494b8aa87cba62bf467bf62 Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Thu, 23 Jun 2022 16:15:37 +0200 Subject: [PATCH 59/90] Update scripts/build_hydro_profile.py Co-authored-by: Martha Frysztacki --- scripts/build_hydro_profile.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 4add4c85..bfba40b2 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -78,8 +78,6 @@ def get_eia_annual_hydro_generation(fn, countries): df.loc["Germany"] = df.filter(like='Germany', axis=0).sum() df.loc["Serbia"] += df.loc["Kosovo"] - df = df.loc[~df.index.str.contains('Former')] - df.drop(["Europe", "Germany, West", "Germany, East"], inplace=True) df.index = cc.convert(df.index, to='iso2') df.index.name = 'countries' From 2e212fb436053af0e2b654b6998a2f20ac763ea1 Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Thu, 23 Jun 2022 16:16:36 +0200 Subject: [PATCH 60/90] Update test/config.test1.yaml Co-authored-by: Martha Frysztacki --- test/config.test1.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 600a3aac..18c01ad2 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -8,7 +8,6 @@ logging: level: INFO format: '%(levelname)s:%(name)s:%(message)s' -results_dir: results/your-run-name scenario: simpl: [''] From 2c5643a5f8edf85130ca88c848e16f628024ae52 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 23 Jun 2022 16:36:22 +0200 Subject: [PATCH 61/90] add_electricity: fix conventional config --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index c37a791d..f2dfdfef 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -600,7 +600,7 @@ if __name__ == "__main__": update_transmission_costs(n, costs, snakemake.config['lines']['length_factor']) - attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers, **snakemake.config["conventional"]) + attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers, snakemake.config.get("conventional", {})) attach_wind_and_solar(n, costs, snakemake.input, renewable_carriers, extendable_carriers, snakemake.config['lines']['length_factor']) From b581e7afc6a6ed0268315c53f9b86c4e11229b2b Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 23 Jun 2022 21:19:41 +0200 Subject: [PATCH 62/90] build_hydro_profiles: revert changes --- scripts/build_hydro_profile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index bfba40b2..4add4c85 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -78,6 +78,8 @@ def get_eia_annual_hydro_generation(fn, countries): df.loc["Germany"] = df.filter(like='Germany', axis=0).sum() df.loc["Serbia"] += df.loc["Kosovo"] + df = df.loc[~df.index.str.contains('Former')] + df.drop(["Europe", "Germany, West", "Germany, East"], inplace=True) df.index = cc.convert(df.index, to='iso2') df.index.name = 'countries' From c42d2bd97d79fde0cf47264f364e5f552b00ea48 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 23 Jun 2022 14:25:30 +0200 Subject: [PATCH 63/90] refactor save to geojson file functionality to allow import/export of empty geodfs --- doc/release_notes.rst | 2 ++ envs/environment.yaml | 2 +- scripts/_helpers.py | 2 ++ scripts/base_network.py | 4 +++- scripts/build_bus_regions.py | 20 ++++++++------------ scripts/build_cutout.py | 2 +- scripts/build_renewable_profiles.py | 6 +++++- scripts/build_shapes.py | 26 ++++++++------------------ scripts/cluster_network.py | 20 +++++++------------- 9 files changed, 37 insertions(+), 47 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 963a1175..39ec0ebe 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -45,6 +45,8 @@ This release is not on the ``master`` branch. It can be used with Upcoming Release ================ +* The workflow now supports to run a selection of countries which do not have any offshore regions assigned. Therefore the offshore technologies need to be disabled, otherwise the workflow will raise an error. + * Add an efficiency factor of 88.55% to offshore wind capacity factors as a proxy for wake losses. More rigorous modelling is `planned `_ [`#277 `_]. diff --git a/envs/environment.yaml b/envs/environment.yaml index f8060de1..039b602a 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -27,7 +27,7 @@ dependencies: - powerplantmatching>=0.5.3 - numpy - pandas - - geopandas + - geopandas>=0.11.0 - xarray - netcdf4 - networkx diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 6e47c053..d77266d8 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -4,7 +4,9 @@ import pandas as pd from pathlib import Path +from collections import OrderedDict +REGION_COLS = ['geometry', 'name', 'x', 'y', 'country'] def configure_logging(snakemake, skip_handlers=False): """ diff --git a/scripts/base_network.py b/scripts/base_network.py index 50ec8e53..1d105225 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -391,7 +391,9 @@ def _set_countries_and_substations(n, config, country_shapes, offshore_shapes): countries = config['countries'] country_shapes = gpd.read_file(country_shapes).set_index('name')['geometry'] - offshore_shapes = gpd.read_file(offshore_shapes).set_index('name')['geometry'] + # reindexing necessary for supporting empty geo-dataframes + offshore_shapes = gpd.read_file(offshore_shapes) + offshore_shapes = offshore_shapes.reindex(columns=['name', 'geometry']).set_index('name')['geometry'] substation_b = buses['symbol'].str.contains('substation|converter station', case=False) def prefer_voltage(x, which): diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index 8003d370..8869c9f4 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -42,7 +42,7 @@ Description """ import logging -from _helpers import configure_logging +from _helpers import configure_logging, REGION_COLS import pypsa import os @@ -55,13 +55,6 @@ from scipy.spatial import Voronoi logger = logging.getLogger(__name__) -def save_to_geojson(s, fn): - if os.path.exists(fn): - os.unlink(fn) - schema = {**gpd.io.file.infer_schema(s), 'geometry': 'Unknown'} - s.to_file(fn, driver='GeoJSON', schema=schema) - - def voronoi_partition_pts(points, outline): """ Compute the polygons of a voronoi partition of `points` within the @@ -120,7 +113,8 @@ if __name__ == "__main__": n = pypsa.Network(snakemake.input.base_network) country_shapes = gpd.read_file(snakemake.input.country_shapes).set_index('name')['geometry'] - offshore_shapes = gpd.read_file(snakemake.input.offshore_shapes).set_index('name')['geometry'] + offshore_shapes = gpd.read_file(snakemake.input.offshore_shapes) + offshore_shapes = offshore_shapes.reindex(columns=REGION_COLS).set_index('name')['geometry'] onshore_regions = [] offshore_regions = [] @@ -151,6 +145,8 @@ if __name__ == "__main__": offshore_regions_c = offshore_regions_c.loc[offshore_regions_c.area > 1e-2] offshore_regions.append(offshore_regions_c) - save_to_geojson(pd.concat(onshore_regions, ignore_index=True), snakemake.output.regions_onshore) - - save_to_geojson(pd.concat(offshore_regions, ignore_index=True), snakemake.output.regions_offshore) + pd.concat(onshore_regions, ignore_index=True).to_file(snakemake.output.regions_onshore) + if offshore_regions: + pd.concat(offshore_regions, ignore_index=True).to_file(snakemake.output.regions_offshore) + else: + offshore_shapes.to_frame().to_file(snakemake.output.regions_offshore) \ No newline at end of file diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 78eafac6..5ab085a1 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -116,7 +116,7 @@ if __name__ == "__main__": # Determine the bounds from bus regions with a buffer of two grid cells onshore = gpd.read_file(snakemake.input.regions_onshore) offshore = gpd.read_file(snakemake.input.regions_offshore) - regions = onshore.append(offshore) + regions = pd.concat([onshore, offshore]) d = max(cutout_params.get('dx', 0.25), cutout_params.get('dy', 0.25))*2 cutout_params['bounds'] = regions.total_bounds + [-d, -d, d, d] elif {'x', 'y'}.issubset(cutout_params): diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 37e1e9de..5db87c78 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -221,7 +221,11 @@ if __name__ == '__main__': client = Client(cluster, asynchronous=True) cutout = atlite.Cutout(snakemake.input['cutout']) - regions = gpd.read_file(snakemake.input.regions).set_index('name').rename_axis('bus') + regions = gpd.read_file(snakemake.input.regions) + assert not regions.empty, (f"List of regions in {snakemake.input.regions} is empty, please " + "disable the corresponding renewable technology") + # do not pull up, set_index does not work if geo dataframe is empty + regions = regions.set_index('name').rename_axis('bus') buses = regions.index excluder = atlite.ExclusionContainer(crs=3035, res=100) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 22aed1fe..09230ddc 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -129,14 +129,15 @@ def eez(country_shapes, eez, country_list): df['name'] = df['ISO_3digit'].map(lambda c: _get_country('alpha_2', alpha_3=c)) s = df.set_index('name').geometry.map(lambda s: _simplify_polys(s, filterremote=False)) s = gpd.GeoSeries({k:v for k,v in s.iteritems() if v.distance(country_shapes[k]) < 1e-3}) + s = s.to_frame("geometry") s.index.name = "name" return s def country_cover(country_shapes, eez_shapes=None): - shapes = list(country_shapes) + shapes = country_shapes if eez_shapes is not None: - shapes += list(eez_shapes) + shapes = pd.concat([shapes, eez_shapes]) europe_shape = unary_union(shapes) if isinstance(europe_shape, MultiPolygon): @@ -203,16 +204,6 @@ def nuts3(country_shapes, nuts3, nuts3pop, nuts3gdp, ch_cantons, ch_popgdp): return df -def save_to_geojson(df, fn): - if os.path.exists(fn): - os.unlink(fn) - if not isinstance(df, gpd.GeoDataFrame): - df = gpd.GeoDataFrame(dict(geometry=df)) - df = df.reset_index() - schema = {**gpd.io.file.infer_schema(df), 'geometry': 'Unknown'} - df.to_file(fn, driver='GeoJSON', schema=schema) - - if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake @@ -220,15 +211,14 @@ if __name__ == "__main__": configure_logging(snakemake) country_shapes = countries(snakemake.input.naturalearth, snakemake.config['countries']) - save_to_geojson(country_shapes, snakemake.output.country_shapes) + country_shapes.reset_index().to_file(snakemake.output.country_shapes) offshore_shapes = eez(country_shapes, snakemake.input.eez, snakemake.config['countries']) - save_to_geojson(offshore_shapes, snakemake.output.offshore_shapes) + offshore_shapes.reset_index().to_file(snakemake.output.offshore_shapes) - europe_shape = country_cover(country_shapes, offshore_shapes) - save_to_geojson(gpd.GeoSeries(europe_shape), snakemake.output.europe_shape) + europe_shape = gpd.GeoDataFrame(geometry=[country_cover(country_shapes, offshore_shapes.geometry)]) + europe_shape.reset_index().to_file(snakemake.output.europe_shape) nuts3_shapes = nuts3(country_shapes, snakemake.input.nuts3, snakemake.input.nuts3pop, snakemake.input.nuts3gdp, snakemake.input.ch_cantons, snakemake.input.ch_popgdp) - - save_to_geojson(nuts3_shapes, snakemake.output.nuts3_shapes) + nuts3_shapes.reset_index().to_file(snakemake.output.nuts3_shapes) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 1d5608e2..7a4daaee 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -122,7 +122,7 @@ Exemplary unsolved network clustered to 37 nodes: """ import logging -from _helpers import configure_logging, update_p_nom_max +from _helpers import configure_logging, update_p_nom_max, REGION_COLS import pypsa import os @@ -303,24 +303,18 @@ def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carr return clustering -def save_to_geojson(s, fn): - if os.path.exists(fn): - os.unlink(fn) - df = s.reset_index() - schema = {**gpd.io.file.infer_schema(df), 'geometry': 'Unknown'} - df.to_file(fn, driver='GeoJSON', schema=schema) - - def cluster_regions(busmaps, input=None, output=None): busmap = reduce(lambda x, y: x.map(y), busmaps[1:], busmaps[0]) for which in ('regions_onshore', 'regions_offshore'): - regions = gpd.read_file(getattr(input, which)).set_index('name') - geom_c = regions.geometry.groupby(busmap).apply(shapely.ops.unary_union) - regions_c = gpd.GeoDataFrame(dict(geometry=geom_c)) + regions = gpd.read_file(getattr(input, which)) + regions = regions.reindex(columns=REGION_COLS).set_index('name') + aggfunc = dict(x="mean", y="mean", country="first") + regions_c = regions.dissolve(busmap, aggfunc=aggfunc) regions_c.index.name = 'name' - save_to_geojson(regions_c, getattr(output, which)) + regions_c = regions_c.reset_index() + regions_c.to_file(getattr(output, which)) def plot_busmap_for_n_clusters(n, n_clusters, fn=None): From 17d7403f20fd5d7e279f33feae9d777a04da371e Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Fri, 24 Jun 2022 15:34:26 +0200 Subject: [PATCH 64/90] build_powerplants: apply suggestions from code review Co-authored-by: Martha Frysztacki --- doc/configtables/electricity.csv | 2 +- scripts/build_powerplants.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index 3ec26106..5abae842 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -14,6 +14,6 @@ max_hours,,, powerplants_filter,--,"use `pandas.query `_ strings here, e.g. Country not in ['Germany']",Filter query for the default powerplant database. custom_powerplants,--,"use `pandas.query `_ strings here, e.g. Country in ['Germany']",Filter query for the custom powerplant database. conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}",List of conventional power plants to include in the model from ``resources/powerplants.csv``. If an included carrier is also listed in `extendable_carriers`, the capacity is taken as a lower bound. -renewable_carriers,--,"Any subset of {solar, offwind-ac, offwind-dc, hydro}",List of renewable generators to include in the model. +renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, hydro}",List of renewable generators to include in the model. estimate_renewable_capacities,,, "-- Fueltype [ppm], e.g. Wind",,"list of fueltypes strings in PyPSA-Eur, e.g. [onwind, offwind-ac, offwind-dc]",converts ppm Fueltype to PyPSA-EUR Fueltype diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index c1ee4127..a5dbf57b 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -131,7 +131,7 @@ if __name__ == "__main__": custom_ppl_query = snakemake.config['electricity']['custom_powerplants'] ppl = add_custom_powerplants(ppl, snakemake.input.custom_powerplants, custom_ppl_query) - countries_wo_ppl = [c for c in countries if c not in ppl.Country.unique()] + countries_wo_ppl = set(countries)-set(ppl.Country.unique()) if countries_wo_ppl: logging.warning(f"No powerplants known in: {', '.join(countries_wo_ppl)}") From 3dbd8d1492b164f3f2f4c126e63cf35837af007a Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 24 Jun 2022 18:31:39 +0200 Subject: [PATCH 65/90] add_electricity: fix read_csv --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index f2dfdfef..9a99d314 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -337,7 +337,7 @@ def attach_conventional_generators(n, costs, ppl, conventional_carriers, extenda # Single value affecting all generators of technology k indiscriminantely of country n.generators.loc[idx, "p_max_pu"] = factors elif isinstance(factors, str): - factors = pd.read_file(factors, index_col=0) + factors = pd.read_csv(factors, index_col=0)['factor'] # Values affecting generators of technology k country-specific # First map generator buses to countries; then map countries to p_max_pu bus_factors = n.buses.country.map(factors) From a1ee747dc612e301477d09c2c50987a49587b231 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 24 Jun 2022 18:37:43 +0200 Subject: [PATCH 66/90] data: add nuclear_eafs --- data/nuclear_eafs.csv | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 data/nuclear_eafs.csv diff --git a/data/nuclear_eafs.csv b/data/nuclear_eafs.csv new file mode 100644 index 00000000..06b5f684 --- /dev/null +++ b/data/nuclear_eafs.csv @@ -0,0 +1,16 @@ +country,factor +BE,0.65 +BG,0.89 +CZ,0.82 +FI,0.92 +FR,0.70 +DE,0.88 +HU,0.90 +NL,0.86 +RO,0.92 +SK,0.89 +SI,0.94 +ES,0.89 +SE,0.82 +CH,0.86 +GB,0.67 \ No newline at end of file From b37c1d98d32274708f8bff8bc3e53bc9f9e283fb Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 24 Jun 2022 18:44:03 +0200 Subject: [PATCH 67/90] update release notes --- doc/release_notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 1825c65c..337f09e9 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -57,7 +57,7 @@ Upcoming Release * A new section ``conventional`` was added to the config file. This section contains configurations for conventional carriers. -* Implement country-specific EAFs for nuclear power plants based on IAEA 2018-2020 reported country averages. These are specified under the ``energy_availibility_factor`` key in the config entry ``conventional`` and specify the static `p_max_pu` values. +* Implement country-specific EAFs for nuclear power plants based on IAEA 2018-2020 reported country averages. These are specified `data/nuclear_eafs.csv` and translate to static `p_max_pu` values. * The powerplants that have been shut down before 2021 are filtered out. From c9c738e96b2240c8522f84a1941b163069c0e351 Mon Sep 17 00:00:00 2001 From: martacki Date: Fri, 24 Jun 2022 20:57:53 +0200 Subject: [PATCH 68/90] clustering strats to configurables: set defaults for yaml-incompatible strats + include in stubaggregation --- config.default.yaml | 1 + config.tutorial.yaml | 1 + scripts/cluster_network.py | 6 +++--- scripts/simplify_network.py | 37 +++++++++++++++++++++++++------------ test/config.test1.yaml | 1 + 5 files changed, 31 insertions(+), 15 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index 7162d449..144f416e 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -31,6 +31,7 @@ clustering: committable: any ramp_limit_up: max ramp_limit_down: max + efficiency: mean snapshots: start: "2013-01-01" diff --git a/config.tutorial.yaml b/config.tutorial.yaml index f18f23f4..31ca7f99 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -31,6 +31,7 @@ clustering: committable: any ramp_limit_up: max ramp_limit_down: max + efficiency:mean snapshots: start: "2013-03-01" diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index fd66b043..93cd89cd 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -264,7 +264,8 @@ def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carr bus_strategies = dict(country=_make_consense("Bus", "country")) bus_strategies.update(aggregation_strategies.get("buses", {})) - generator_strategies = aggregation_strategies.get("generators", {"p_nom_max": "sum"}) + generator_strategies = {'build_year': lambda x: 0, 'lifetime': lambda x: np.inf} + generator_strategies.update(aggregation_strategies.get("generators", {})) if not isinstance(custom_busmap, pd.Series): busmap = busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights, algorithm) @@ -366,7 +367,6 @@ if __name__ == "__main__": ) return v aggregation_strategies = snakemake.config["clustering"].get("aggregation_strategies", {}) - aggregation_strategies = {} # translate str entries of aggregation_strategies to pd.Series functions: aggregation_strategies = { p: {k: getattr(pd.Series, v) for k,v in aggregation_strategies[p].items()} @@ -383,7 +383,7 @@ if __name__ == "__main__": line_length_factor, aggregation_strategies, snakemake.config['solving']['solver']['name'], "kmeans", hvac_overhead_cost, focus_weights) - + update_p_nom_max(clustering.network) clustering.network.export_to_netcdf(snakemake.output.network) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 52e0c815..0fda5c77 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -189,7 +189,10 @@ def _adjust_capital_costs_using_connection_costs(n, connection_costs_to_bus, out -def _aggregate_and_move_components(n, busmap, connection_costs_to_bus, output, aggregate_one_ports={"Load", "StorageUnit"}): +def _aggregate_and_move_components(n, busmap, connection_costs_to_bus, output, + aggregate_one_ports={"Load", "StorageUnit"}, + aggregation_strategies=dict()): + def replace_components(n, c, df, pnl): n.mremove(c, n.df(c).index) @@ -200,7 +203,12 @@ def _aggregate_and_move_components(n, busmap, connection_costs_to_bus, output, a _adjust_capital_costs_using_connection_costs(n, connection_costs_to_bus, output) - generators, generators_pnl = aggregategenerators(n, busmap, custom_strategies={'p_nom_min': np.sum}) + generator_strategies = {'build_year': lambda x: 0, 'lifetime': lambda x: np.inf} + generator_strategies.update(aggregation_strategies.get("generators", {})) + + generators, generators_pnl = aggregategenerators( + n, busmap, custom_strategies=generator_strategies + ) replace_components(n, "Generator", generators, generators_pnl) for one_port in aggregate_one_ports: @@ -214,7 +222,7 @@ def _aggregate_and_move_components(n, busmap, connection_costs_to_bus, output, a n.mremove(c, df.index[df.bus0.isin(buses_to_del) | df.bus1.isin(buses_to_del)]) -def simplify_links(n, costs, config, output): +def simplify_links(n, costs, config, output, aggregation_strategies=dict()): ## Complex multi-node links are folded into end-points logger.info("Simplifying connected link components") @@ -306,17 +314,19 @@ def simplify_links(n, costs, config, output): logger.debug("Collecting all components using the busmap") - _aggregate_and_move_components(n, busmap, connection_costs_to_bus, output) + _aggregate_and_move_components(n, busmap, connection_costs_to_bus, output, + aggregation_strategies=aggregation_strategies) return n, busmap -def remove_stubs(n, costs, config, output): +def remove_stubs(n, costs, config, output, aggregation_strategies=dict()): logger.info("Removing stubs") busmap = busmap_by_stubs(n) # ['country']) connection_costs_to_bus = _compute_connection_costs_to_bus(n, busmap, costs, config) - _aggregate_and_move_components(n, busmap, connection_costs_to_bus, output) + _aggregate_and_move_components(n, busmap, connection_costs_to_bus, output, + aggregation_strategies=aggregation_strategies) return n, busmap @@ -345,13 +355,14 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None): busmap = n.buses.index.to_series() busmap.loc[buses_i] = dist.idxmin(1) - # default aggregation strategies must be specified within the function, otherwise (when defaults - # are passed in the function's definition) they get lost in case custom values for different - # variables are specified in the config. + # default aggregation strategies that cannot be defined in .yaml format must be specified within + # the function, otherwise (when defaults are passed in the function's definition) they get lost + # in case custom values for different variables are specified in the config. bus_strategies = dict(country=_make_consense("Bus", "country")) bus_strategies.update(aggregation_strategies.get("buses", {})) - generator_strategies = aggregation_strategies.get("generators", {"p_nom_max": "sum"}) + generator_strategies = {'build_year': lambda x: 0, 'lifetime': lambda x: np.inf} + generator_strategies.update(aggregation_strategies.get("generators", {})) clustering = get_clustering_from_busmap(n, busmap, bus_strategies=bus_strategies, @@ -403,9 +414,11 @@ if __name__ == "__main__": technology_costs = load_costs(snakemake.input.tech_costs, snakemake.config['costs'], snakemake.config['electricity'], Nyears) - n, simplify_links_map = simplify_links(n, technology_costs, snakemake.config, snakemake.output) + n, simplify_links_map = simplify_links(n, technology_costs, snakemake.config, snakemake.output, + aggregation_strategies) - n, stub_map = remove_stubs(n, technology_costs, snakemake.config, snakemake.output) + n, stub_map = remove_stubs(n, technology_costs, snakemake.config, snakemake.output, + aggregation_strategies=aggregation_strategies) busmaps = [trafo_map, simplify_links_map, stub_map] diff --git a/test/config.test1.yaml b/test/config.test1.yaml index e3f39ab5..6d626f7e 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -30,6 +30,7 @@ clustering: committable: any ramp_limit_up: max ramp_limit_down: max + efficiency: mean snapshots: start: "2013-03-01" From 8349e85252d2e296d9ba45f4bbfd60e621a68599 Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Mon, 27 Jun 2022 11:48:45 +0200 Subject: [PATCH 69/90] Apply suggestions from code review Co-authored-by: Fabian Neumann --- doc/release_notes.rst | 8 +++++--- scripts/add_electricity.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 300bef88..57c0a2f0 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -21,7 +21,7 @@ Upcoming Release correction factor for solar PV capacity factors by default while satellite data is used. A correction factor of 0.854337 is recommended if reanalysis data like ERA5 is used. -* Resource definitions for memory usage now follow [Snakemake standard resource definition](https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#standard-resources) ```mem_mb`` rather than ``mem``. +* Resource definitions for memory usage now follow `Snakemake standard resource definition `_ ``mem_mb`` rather than ``mem``. * Network building is made deterministic by supplying a fixed random state to network clustering routines. @@ -51,13 +51,13 @@ Upcoming Release * Add operational reserve margin constraint analogous to `GenX implementation `_. Can be activated with config setting ``electricity: operational_reserve:``. -* Add function to add global constraint on use of gas in :mod:`prepare_network`. This can be activated with `electricity: gaslimit:` given in MWh. +* Add function to add global constraint on use of gas in :mod:`prepare_network`. This can be activated by including the keyword ``CH4L`` in the ``{opts}`` wildcard which enforces the limit set in ``electricity: gaslimit:`` given in MWh thermal. Alternatively, it is possible to append a number in the `{opts}` wildcard, e.g. `CH4L200` which limits the gas use to 200 TWh thermal. * Add configuration option to implement Energy Availability Factors (EAFs) for conventional generation technologies. * A new section ``conventional`` was added to the config file. This section contains configurations for conventional carriers. -* Implement country-specific EAFs for nuclear power plants based on IAEA 2018-2020 reported country averages. These are specified `data/nuclear_eafs.csv` and translate to static `p_max_pu` values. +* Implement country-specific EAFs for nuclear power plants based on IAEA 2018-2020 reported country averages. These are specified ``data/nuclear_eafs.csv`` and translate to static ``p_max_pu`` values. * The powerplants that have been shut down before 2021 are filtered out. @@ -84,7 +84,9 @@ Upcoming Release * Cache data and cutouts folders. This cache will be updated weekly. * Add rule to automatically retrieve Natura2000 natural protection areas. Switch of file format to GPKG. +* Add option to set CO2 emission prices through `{opts}` wildcard: `Ep`, e.g. `Ep180`, will set the EUR/tCO2 price. +* Add option to alter marginal costs of a carrier through `{opts}` wildcard: `+m`, e.g. `gas+m2.5`, will multiply the default marginal cost for gas by factor 2.5. Synchronisation Release - Ukraine and Moldova (17th March 2022) =============================================================== diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 9a99d314..ea95fd94 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -324,7 +324,7 @@ def attach_conventional_generators(n, costs, ppl, conventional_carriers, extenda marginal_cost=ppl.marginal_cost, capital_cost=ppl.capital_cost, build_year=ppl.datein.fillna(0).astype(int), - lifetime=(ppl.dateout - ppl.datein).fillna(9999).astype(int), + lifetime=(ppl.dateout - ppl.datein).fillna(np.inf), ) for carrier in conventional_config: @@ -446,7 +446,7 @@ def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **con def attach_extendable_generators(n, costs, ppl, carriers): - logger.warning("The function `attach_extendable_generators` is deprecated in v0.0.5.") + logger.warning("The function `attach_extendable_generators` is deprecated in v0.5.0.") _add_missing_carriers_from_costs(n, costs, carriers) for tech in carriers: From 2dfa2753ba8b5de6df56d6e6f9b278aa32047d55 Mon Sep 17 00:00:00 2001 From: martacki Date: Mon, 27 Jun 2022 13:59:04 +0200 Subject: [PATCH 70/90] clustering strats to configurables: config spacing --- config.tutorial.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 31ca7f99..edf0091c 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -31,7 +31,7 @@ clustering: committable: any ramp_limit_up: max ramp_limit_down: max - efficiency:mean + efficiency: mean snapshots: start: "2013-03-01" From a3af137b74d6e505e2dae32fc7cea2b2226f834c Mon Sep 17 00:00:00 2001 From: martacki Date: Mon, 27 Jun 2022 14:18:47 +0200 Subject: [PATCH 71/90] clustering strats to configurables: move duplicate code to _helpers script & import --- scripts/_helpers.py | 16 ++++++++++++++++ scripts/cluster_network.py | 8 ++------ scripts/simplify_network.py | 14 +++----------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 6e47c053..3c3e6ac7 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -210,6 +210,22 @@ def progress_retrieve(url, file): urllib.request.urlretrieve(url, file, reporthook=dlProgress) +def get_aggregation_strategies(aggregation_strategies): + # default aggregation strategies that cannot be defined in .yaml format must be specified within + # the function, otherwise (when defaults are passed in the function's definition) they get lost + # when custom values are specified in the config. + + import numpy as np + from pypsa.networkclustering import _make_consense + + bus_strategies = dict(country=_make_consense("Bus", "country")) + bus_strategies.update(aggregation_strategies.get("buses", {})) + + generator_strategies = {'build_year': lambda x: 0, 'lifetime': lambda x: np.inf} + generator_strategies.update(aggregation_strategies.get("generators", {})) + + return bus_strategies, generator_strategies + def mock_snakemake(rulename, **wildcards): """ diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 93cd89cd..6186a00e 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -121,7 +121,7 @@ Exemplary unsolved network clustered to 37 nodes: """ import logging -from _helpers import configure_logging, update_p_nom_max +from _helpers import configure_logging, update_p_nom_max, get_aggregation_strategies import pypsa import os @@ -261,11 +261,7 @@ def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carr line_length_factor=1.25, aggregation_strategies=dict(), solver_name="cbc", algorithm="kmeans", extended_link_costs=0, focus_weights=None): - bus_strategies = dict(country=_make_consense("Bus", "country")) - bus_strategies.update(aggregation_strategies.get("buses", {})) - - generator_strategies = {'build_year': lambda x: 0, 'lifetime': lambda x: np.inf} - generator_strategies.update(aggregation_strategies.get("generators", {})) + bus_strategies, generator_strategies = get_aggregation_strategies(aggregation_strategies) if not isinstance(custom_busmap, pd.Series): busmap = busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights, algorithm) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 0fda5c77..37e7e698 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -83,7 +83,7 @@ The rule :mod:`simplify_network` does up to four things: """ import logging -from _helpers import configure_logging, update_p_nom_max +from _helpers import configure_logging, update_p_nom_max, get_aggregation_strategies from cluster_network import clustering_for_n_clusters, cluster_regions from add_electricity import load_costs @@ -203,8 +203,7 @@ def _aggregate_and_move_components(n, busmap, connection_costs_to_bus, output, _adjust_capital_costs_using_connection_costs(n, connection_costs_to_bus, output) - generator_strategies = {'build_year': lambda x: 0, 'lifetime': lambda x: np.inf} - generator_strategies.update(aggregation_strategies.get("generators", {})) + _, generator_strategies = get_aggregation_strategies(aggregation_strategies) generators, generators_pnl = aggregategenerators( n, busmap, custom_strategies=generator_strategies @@ -355,14 +354,7 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None): busmap = n.buses.index.to_series() busmap.loc[buses_i] = dist.idxmin(1) - # default aggregation strategies that cannot be defined in .yaml format must be specified within - # the function, otherwise (when defaults are passed in the function's definition) they get lost - # in case custom values for different variables are specified in the config. - bus_strategies = dict(country=_make_consense("Bus", "country")) - bus_strategies.update(aggregation_strategies.get("buses", {})) - - generator_strategies = {'build_year': lambda x: 0, 'lifetime': lambda x: np.inf} - generator_strategies.update(aggregation_strategies.get("generators", {})) + bus_strategies, generator_strategies = get_aggregation_strategies(aggregation_strategies) clustering = get_clustering_from_busmap(n, busmap, bus_strategies=bus_strategies, From b56d1f6f4d445536708779f3c886ddfec647aabf Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 27 Jun 2022 17:35:19 +0200 Subject: [PATCH 72/90] conventional config section: update to more general attribute assignment scheme --- Snakefile | 5 ++-- config.default.yaml | 2 +- ...{nuclear_eafs.csv => nuclear_p_max_pu.csv} | 0 doc/configuration.rst | 13 ++++++++++- doc/release_notes.rst | 6 ++--- scripts/add_electricity.py | 23 +++++++++++-------- 6 files changed, 32 insertions(+), 17 deletions(-) rename data/{nuclear_eafs.csv => nuclear_p_max_pu.csv} (100%) diff --git a/Snakefile b/Snakefile index af2e6e90..71c6bc21 100644 --- a/Snakefile +++ b/Snakefile @@ -174,7 +174,7 @@ rule build_renewable_profiles: input: base_network="networks/base.nc", corine="data/bundle/corine/g250_clc06_V18_5.tif", - natura=lambda w: ("data/Natura2000_end2020.gpkg" + natura=lambda w: ("resources/natura.tiff" if config["renewable"][w.technology]["natura"] else []), gebco=lambda w: ("data/bundle/GEBCO_2014_2D.nc" @@ -217,7 +217,8 @@ rule add_electricity: load='resources/load.csv', nuts3_shapes='resources/nuts3_shapes.geojson', **{f"profile_{tech}": f"resources/profile_{tech}.nc" - for tech in config['renewable']} + for tech in config['renewable']}, + **{"conventional_{carrier}_{attrs}": fn for carrier in config.get('conventional', {None: {}}).values() for fn in carrier.values() if str(fn).startswith("data/")}, output: "networks/elec.nc" log: "logs/add_electricity.log" benchmark: "benchmarks/add_electricity" diff --git a/config.default.yaml b/config.default.yaml index 177c5e74..a67562c6 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -187,7 +187,7 @@ renewable: conventional: nuclear: - energy_availability_factors: "data/nuclear-eafs.csv" # float of file name + p_max_pu: "data/nuclear_p_max_pu.csv" # float of file name lines: types: diff --git a/data/nuclear_eafs.csv b/data/nuclear_p_max_pu.csv similarity index 100% rename from data/nuclear_eafs.csv rename to data/nuclear_p_max_pu.csv diff --git a/doc/configuration.rst b/doc/configuration.rst index 67d25228..c332ea7d 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -171,7 +171,7 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia .. literalinclude:: ../config.default.yaml :language: yaml :start-at: hydro: - :end-before: lines: + :end-before: conventional: .. csv-table:: :header-rows: 1 @@ -180,6 +180,17 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia .. _lines_cf: +``conventional`` +============= + +Define additional generator attribute for conventional carrier types. If a scalar value is given it is applied to all generators. However if a string starting with "data/" is given, the value is interpreted as a path to a csv file with country specific values. Then, the values are read in and applied to all generators of the given carrier in the given country. Note that the value(s) overwrite the existing values in the corresponding section of the ``generators`` dataframe. + +.. literalinclude:: ../config.default.yaml + :language: yaml + :start-at: conventional: + :end-before: lines: + + ``lines`` ============= diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 57c0a2f0..3addf3ab 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -53,11 +53,11 @@ Upcoming Release * Add function to add global constraint on use of gas in :mod:`prepare_network`. This can be activated by including the keyword ``CH4L`` in the ``{opts}`` wildcard which enforces the limit set in ``electricity: gaslimit:`` given in MWh thermal. Alternatively, it is possible to append a number in the `{opts}` wildcard, e.g. `CH4L200` which limits the gas use to 200 TWh thermal. -* Add configuration option to implement Energy Availability Factors (EAFs) for conventional generation technologies. - * A new section ``conventional`` was added to the config file. This section contains configurations for conventional carriers. -* Implement country-specific EAFs for nuclear power plants based on IAEA 2018-2020 reported country averages. These are specified ``data/nuclear_eafs.csv`` and translate to static ``p_max_pu`` values. +* Add configuration option to implement arbitrary generator attributes for conventional generation technologies. + +* Implement country-specific Energy Availability Factors (EAFs) for nuclear power plants based on IAEA 2018-2020 reported country averages. These are specified ``data/nuclear_p_max_pu.csv`` and translate to static ``p_max_pu`` values. * The powerplants that have been shut down before 2021 are filtered out. diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index ea95fd94..88c2ce66 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -331,17 +331,20 @@ def attach_conventional_generators(n, costs, ppl, conventional_carriers, extenda # Generators with technology affected idx = n.generators.query("carrier == @carrier").index - factors = conventional_config[carrier].get("energy_availability_factors") - if isinstance(factors, float): - # Single value affecting all generators of technology k indiscriminantely of country - n.generators.loc[idx, "p_max_pu"] = factors - elif isinstance(factors, str): - factors = pd.read_csv(factors, index_col=0)['factor'] - # Values affecting generators of technology k country-specific - # First map generator buses to countries; then map countries to p_max_pu - bus_factors = n.buses.country.map(factors) - n.generators.p_max_pu.update(n.generators.loc[idx].bus.map(bus_factors).dropna()) + for key in list(set(conventional_carriers[carrier]) & set(n.generators)): + + values = conventional_config[carrier][key] + + if isinstance(values, str) and str(values).startswith("data/"): + # Values affecting generators of technology k country-specific + # First map generator buses to countries; then map countries to p_max_pu + values = pd.read_csv(values, index_col=0).iloc[:, 0] + bus_values = n.buses.country.map(values) + n.generators[key].update(n.generators.loc[idx].bus.map(bus_values).dropna()) + else: + # Single value affecting all generators of technology k indiscriminantely of country + n.generators.loc[idx, key] = values From 51de606aab6eddcba8d121873f779c4ce995afa8 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 27 Jun 2022 19:00:41 +0200 Subject: [PATCH 73/90] Revert "remove build/retrieve natura raster, directly use shapefile" --- .github/workflows/ci.yaml | 23 ++--- Snakefile | 27 ++++-- config.default.yaml | 2 + config.tutorial.yaml | 10 ++- doc/configtables/toplevel.csv | 2 + doc/preparation.rst | 2 + doc/preparation/build_natura_raster.rst | 39 +++++++++ doc/preparation/build_renewable_profiles.rst | 3 + doc/preparation/retrieve.rst | 27 ++++++ doc/release_notes.rst | 5 -- doc/tutorial.rst | 4 +- envs/environment.yaml | 1 - scripts/build_natura_raster.py | 89 ++++++++++++++++++++ scripts/build_renewable_profiles.py | 4 +- test/config.test1.yaml | 2 + 15 files changed, 200 insertions(+), 40 deletions(-) create mode 100644 doc/preparation/build_natura_raster.rst create mode 100644 scripts/build_natura_raster.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d447a56f..c753deab 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,12 +13,13 @@ on: branches: - master pull_request: + branches: + - master schedule: - cron: "0 5 * * TUE" env: - CONDA_CACHE_NUMBER: 1 # Change this value to manually reset the environment cache - DATA_CACHE_NUMBER: 1 + CACHE_NUMBER: 1 # Change this value to manually reset the environment cache jobs: build: @@ -65,26 +66,16 @@ jobs: miniforge-version: latest activate-environment: pypsa-eur use-mamba: true - - - name: Set cache dates - run: | - echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV - echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV - - - name: Cache data and cutouts folders - uses: actions/cache@v3 - with: - path: | - data - cutouts - key: data-cutouts-${{ env.WEEK }}-${{ env.DATA_CACHE_NUMBER }} + + - name: Set cache date + run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV - name: Create environment cache uses: actions/cache@v2 id: cache with: path: ${{ matrix.prefix }} - key: ${{ matrix.label }}-conda-${{ hashFiles('envs/environment.yaml') }}-${{ env.DATE }}-${{ env.CONDA_CACHE_NUMBER }} + key: ${{ matrix.label }}-conda-${{ hashFiles('envs/environment.yaml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }} - name: Update environment due to outdated or unavailable cache run: mamba env update -n pypsa-eur -f envs/environment.yaml diff --git a/Snakefile b/Snakefile index 42ecc45b..7678a401 100644 --- a/Snakefile +++ b/Snakefile @@ -67,12 +67,6 @@ if config['enable'].get('retrieve_databundle', True): script: 'scripts/retrieve_databundle.py' -rule retrieve_natura_data: - input: HTTP.remote("sdi.eea.europa.eu/datashare/s/H6QGCybMdLLnywo/download", additional_request_string="?path=%2FNatura2000_end2020_gpkg&files=Natura2000_end2020.gpkg", static=True) - output: "data/Natura2000_end2020.gpkg" - run: move(input[0], output[0]) - - rule retrieve_load_data: input: HTTP.remote("data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv", keep_local=True, static=True) output: "data/load_raw.csv" @@ -171,13 +165,28 @@ if config['enable'].get('retrieve_cutout', True): run: move(input[0], output[0]) +if config['enable'].get('build_natura_raster', False): + rule build_natura_raster: + input: + natura="data/bundle/natura/Natura2000_end2015.shp", + cutouts=expand("cutouts/{cutouts}.nc", **config['atlite']) + output: "resources/natura.tiff" + log: "logs/build_natura_raster.log" + script: "scripts/build_natura_raster.py" + + +if config['enable'].get('retrieve_natura_raster', True): + rule retrieve_natura_raster: + input: HTTP.remote("zenodo.org/record/4706686/files/natura.tiff", keep_local=True, static=True) + output: "resources/natura.tiff" + run: move(input[0], output[0]) + + rule build_renewable_profiles: input: base_network="networks/base.nc", corine="data/bundle/corine/g250_clc06_V18_5.tif", - natura=lambda w: ("data/Natura2000_end2020.gpkg" - if config["renewable"][w.technology]["natura"] - else []), + natura="resources/natura.tiff", gebco=lambda w: ("data/bundle/GEBCO_2014_2D.nc" if "max_depth" in config["renewable"][w.technology].keys() else []), diff --git a/config.default.yaml b/config.default.yaml index 7e6a1f78..144f416e 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -43,6 +43,8 @@ enable: retrieve_databundle: true build_cutout: false retrieve_cutout: true + build_natura_raster: false + retrieve_natura_raster: true custom_busmap: false electricity: diff --git a/config.tutorial.yaml b/config.tutorial.yaml index ba0bb01a..edf0091c 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -43,6 +43,8 @@ enable: retrieve_databundle: true build_cutout: false retrieve_cutout: true + build_natura_raster: false + retrieve_natura_raster: true custom_busmap: false electricity: @@ -87,7 +89,7 @@ renewable: 24, 25, 26, 27, 28, 29, 31, 32] distance: 1000 distance_grid_codes: [1, 2, 3, 4, 5, 6] - natura: false + natura: true potential: simple # or conservative clip_p_max_pu: 1.e-2 offwind-ac: @@ -98,7 +100,7 @@ renewable: capacity_per_sqkm: 3 # correction_factor: 0.93 corine: [44, 255] - natura: false + natura: true max_shore_distance: 30000 potential: simple # or conservative clip_p_max_pu: 1.e-2 @@ -111,7 +113,7 @@ renewable: capacity_per_sqkm: 3 # correction_factor: 0.93 corine: [44, 255] - natura: false + natura: true min_shore_distance: 30000 potential: simple # or conservative clip_p_max_pu: 1.e-2 @@ -133,7 +135,7 @@ renewable: # correction_factor: 0.854337 corine: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 26, 31, 32] - natura: false + natura: true potential: simple # or conservative clip_p_max_pu: 1.e-2 diff --git a/doc/configtables/toplevel.csv b/doc/configtables/toplevel.csv index 8965a0bc..b7f39d05 100644 --- a/doc/configtables/toplevel.csv +++ b/doc/configtables/toplevel.csv @@ -12,4 +12,6 @@ enable,,, -- retrieve_databundle,bool,"{true, false}","Switch to retrieve databundle from zenodo via the rule :mod:`retrieve_databundle` or whether to keep a custom databundle located in the corresponding folder." -- build_cutout,bool,"{true, false}","Switch to enable the building of cutouts via the rule :mod:`build_cutout`." -- retrieve_cutout,bool,"{true, false}","Switch to enable the retrieval of cutouts from zenodo with :mod:`retrieve_cutout`." +-- build_natura_raster,bool,"{true, false}","Switch to enable the creation of the raster ``natura.tiff`` via the rule :mod:`build_natura_raster`." +-- retrieve_natura_raster,bool,"{true, false}","Switch to enable the retrieval of ``natura.tiff`` from zenodo with :mod:`retrieve_natura_raster`." -- custom_busmap,bool,"{true, false}","Switch to enable the use of custom busmaps in rule :mod:`cluster_network`. If activated the rule looks for provided busmaps at ``data/custom_busmap_elec_s{simpl}_{clusters}.csv`` which should have the same format as ``resources/busmap_elec_s{simpl}_{clusters}.csv``, i.e. the index should contain the buses of ``networks/elec_s{simpl}.nc``." diff --git a/doc/preparation.rst b/doc/preparation.rst index 7f42190c..dba5e981 100644 --- a/doc/preparation.rst +++ b/doc/preparation.rst @@ -27,6 +27,7 @@ With these and the externally extracted ENTSO-E online map topology Then the process continues by calculating conventional power plant capacities, potentials, and per-unit availability time series for variable renewable energy carriers and hydro power plants with the following rules: - :mod:`build_powerplants` for today's thermal power plant capacities using `powerplantmatching `_ allocating these to the closest substation for each powerplant, +- :mod:`build_natura_raster` for rasterising NATURA2000 natural protection areas, - :mod:`build_renewable_profiles` for the hourly capacity factors and installation potentials constrained by land-use in each substation's Voronoi cell for PV, onshore and offshore wind, and - :mod:`build_hydro_profile` for the hourly per-unit hydro power availability time series. @@ -40,6 +41,7 @@ together into a detailed PyPSA network stored in ``networks/elec.nc``. preparation/build_shapes preparation/build_load_data preparation/build_cutout + preparation/build_natura_raster preparation/prepare_links_p_nom preparation/base_network preparation/build_bus_regions diff --git a/doc/preparation/build_natura_raster.rst b/doc/preparation/build_natura_raster.rst new file mode 100644 index 00000000..e3ec4364 --- /dev/null +++ b/doc/preparation/build_natura_raster.rst @@ -0,0 +1,39 @@ +.. + SPDX-FileCopyrightText: 2019-2020 The PyPSA-Eur Authors + + SPDX-License-Identifier: CC-BY-4.0 + +.. _natura: + +Rule ``build_natura_raster`` +=============================== + +.. graphviz:: + :align: center + + digraph snakemake_dag { + graph [bgcolor=white, + margin=0, + size="8,5" + ]; + node [fontname=sans, + fontsize=10, + penwidth=2, + shape=box, + style=rounded + ]; + edge [color=grey, + penwidth=2 + ]; + 9 [color="0.22 0.6 0.85", + label=build_renewable_profiles]; + 12 [color="0.31 0.6 0.85", + fillcolor=gray, + label=build_natura_raster, + style=filled]; + 12 -> 9; + } + +| + +.. automodule:: build_natura_raster diff --git a/doc/preparation/build_renewable_profiles.rst b/doc/preparation/build_renewable_profiles.rst index adc4d6ca..27e61583 100644 --- a/doc/preparation/build_renewable_profiles.rst +++ b/doc/preparation/build_renewable_profiles.rst @@ -41,6 +41,9 @@ Rule ``build_renewable_profiles`` 8 [color="0.00 0.6 0.85", label=build_shapes]; 8 -> 9; + 12 [color="0.31 0.6 0.85", + label=build_natura_raster]; + 12 -> 9; 13 [color="0.56 0.6 0.85", label=build_cutout]; 13 -> 9; diff --git a/doc/preparation/retrieve.rst b/doc/preparation/retrieve.rst index 21187924..42479284 100644 --- a/doc/preparation/retrieve.rst +++ b/doc/preparation/retrieve.rst @@ -50,3 +50,30 @@ The :ref:`tutorial` uses a smaller cutout than required for the full model (30 M .. seealso:: For details see :mod:`build_cutout` and read the `atlite documentation `_. + + +Rule ``retrieve_natura_raster`` +------------------------------- + +.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.4706686.svg + :target: https://doi.org/10.5281/zenodo.4706686 + +This rule, as a substitute for :mod:`build_natura_raster`, downloads an already rasterized version (`natura.tiff `_) of `Natura 2000 `_ natural protection areas to reduce computation times. The file is placed into the ``resources`` sub-directory. + +**Relevant Settings** + +.. code:: yaml + + enable: + build_natura_raster: + +.. seealso:: + Documentation of the configuration file ``config.yaml`` at + :ref:`toplevel_cf` + +**Outputs** + +- ``resources/natura.tiff``: Rasterized version of `Natura 2000 `_ natural protection areas to reduce computation times. + +.. seealso:: + For details see :mod:`build_natura_raster`. diff --git a/doc/release_notes.rst b/doc/release_notes.rst index df7af28e..c000a046 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -62,11 +62,6 @@ Upcoming Release * New network topology extracted from the ENTSO-E interactive map. -* Remove rules to build or retrieve rasterized NATURA 2000 dataset. Renewable potential calculation now directly uses the shapefiles. - -* Cache data and cutouts folders. This cache will be updated weekly. - -* Add rule to automatically retrieve Natura2000 natural protection areas. Switch of file format to GPKG. * The unused argument ``simple_hvdc_costs`` in :mod:`add_electricity` was removed. * Iterative solving with impedance updates is skipped if there are no expandable lines. diff --git a/doc/tutorial.rst b/doc/tutorial.rst index 1c247f8e..c37abb39 100644 --- a/doc/tutorial.rst +++ b/doc/tutorial.rst @@ -35,8 +35,8 @@ To run the tutorial, use this as your configuration file ``config.yaml``. .../pypsa-eur % cp config.tutorial.yaml config.yaml -This configuration is set to download a reduced data set via the rules :mod:`retrieve_databundle` -and :mod:`retrieve_cutout` totalling at less than 250 MB. +This configuration is set to download a reduced data set via the rules :mod:`retrieve_databundle`, +:mod:`retrieve_natura_raster`, :mod:`retrieve_cutout` totalling at less than 250 MB. The full set of data dependencies would consume 5.3 GB. For more information on the data dependencies of PyPSA-Eur, continue reading :ref:`data`. diff --git a/envs/environment.yaml b/envs/environment.yaml index fd6dff49..f8060de1 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -45,7 +45,6 @@ dependencies: # GIS dependencies: - cartopy - descartes - - fiona # explicit for Windows - rasterio<=1.2.9 # 1.2.10 creates error https://github.com/PyPSA/atlite/issues/238 # PyPSA-Eur-Sec Dependencies diff --git a/scripts/build_natura_raster.py b/scripts/build_natura_raster.py new file mode 100644 index 00000000..7fa9d544 --- /dev/null +++ b/scripts/build_natura_raster.py @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: : 2017-2020 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + +""" +Rasters the vector data of the `Natura 2000 `_ natural protection areas onto all cutout regions. + +Relevant Settings +----------------- + +.. code:: yaml + + renewable: + {technology}: + cutout: + +.. seealso:: + Documentation of the configuration file ``config.yaml`` at + :ref:`renewable_cf` + +Inputs +------ + +- ``data/bundle/natura/Natura2000_end2015.shp``: `Natura 2000 `_ natural protection areas. + + .. image:: ../img/natura.png + :scale: 33 % + +Outputs +------- + +- ``resources/natura.tiff``: Rasterized version of `Natura 2000 `_ natural protection areas to reduce computation times. + + .. image:: ../img/natura.png + :scale: 33 % + +Description +----------- + +""" + +import logging +from _helpers import configure_logging + +import atlite +import geopandas as gpd +import rasterio as rio +from rasterio.features import geometry_mask +from rasterio.warp import transform_bounds + +logger = logging.getLogger(__name__) + + +def determine_cutout_xXyY(cutout_name): + cutout = atlite.Cutout(cutout_name) + assert cutout.crs.to_epsg() == 4326 + x, X, y, Y = cutout.extent + dx, dy = cutout.dx, cutout.dy + return [x - dx/2., X + dx/2., y - dy/2., Y + dy/2.] + + +def get_transform_and_shape(bounds, res): + left, bottom = [(b // res)* res for b in bounds[:2]] + right, top = [(b // res + 1) * res for b in bounds[2:]] + shape = int((top - bottom) // res), int((right - left) / res) + transform = rio.Affine(res, 0, left, 0, -res, top) + return transform, shape + + +if __name__ == "__main__": + if 'snakemake' not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake('build_natura_raster') + configure_logging(snakemake) + + cutouts = snakemake.input.cutouts + xs, Xs, ys, Ys = zip(*(determine_cutout_xXyY(cutout) for cutout in cutouts)) + bounds = transform_bounds(4326, 3035, min(xs), min(ys), max(Xs), max(Ys)) + transform, out_shape = get_transform_and_shape(bounds, res=100) + + # adjusted boundaries + shapes = gpd.read_file(snakemake.input.natura).to_crs(3035) + raster = ~geometry_mask(shapes.geometry, out_shape[::-1], transform) + raster = raster.astype(rio.uint8) + + with rio.open(snakemake.output[0], 'w', driver='GTiff', dtype=rio.uint8, + count=1, transform=transform, crs=3035, compress='lzw', + width=raster.shape[1], height=raster.shape[0]) as dst: + dst.write(raster, indexes=1) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index a49df1f5..37e1e9de 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -227,9 +227,7 @@ if __name__ == '__main__': excluder = atlite.ExclusionContainer(crs=3035, res=100) if config['natura']: - mask = regions.to_crs(3035).buffer(0) # buffer to avoid invalid geometry - natura = gpd.read_file(snakemake.input.natura, mask=mask) - excluder.add_geometry(natura.geometry) + excluder.add_raster(snakemake.input.natura, nodata=0, allow_no_overlap=True) corine = config.get("corine", {}) if "grid_codes" in corine: diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 64b169a4..6d626f7e 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -42,6 +42,8 @@ enable: retrieve_databundle: true build_cutout: false retrieve_cutout: true + build_natura_raster: false + retrieve_natura_raster: true custom_busmap: false electricity: From 29fe0fb7fb8ab34cde94dc01ca947cece2ed924c Mon Sep 17 00:00:00 2001 From: martacki Date: Mon, 27 Jun 2022 21:02:45 +0200 Subject: [PATCH 74/90] hierarchical clustering: account for changes from merging master --- scripts/cluster_network.py | 1 + scripts/simplify_network.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index b1bf759d..59641a7e 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -12,6 +12,7 @@ Relevant Settings .. code:: yaml clustering: + cluster_network: aggregation_strategies: focus_weights: diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 7e258966..0b31e5c6 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -14,7 +14,8 @@ Relevant Settings .. code:: yaml clustering: - simplify: + simplify_network: + cluster_network: aggregation_strategies: costs: @@ -415,7 +416,7 @@ if __name__ == "__main__": busmaps = [trafo_map, simplify_links_map, stub_map] cluster_config = snakemake.config.get('clustering', {}).get('simplify_network', {}) - if cluster_config.get('clustering', {}).get('simplify', {}).get('to_substations', False): + if cluster_config.get('clustering', {}).get('simplify_network', {}).get('to_substations', False): n, substation_map = aggregate_to_substations(n, aggregation_strategies) busmaps.append(substation_map) @@ -429,7 +430,7 @@ if __name__ == "__main__": for carrier in carriers: buses_i = list(set(n.buses.index)-set(n.generators.query("carrier == @carrier").bus)) logger.info(f'clustering preparaton (hac): aggregating {len(buses_i)} buses of type {carrier}.') - n, busmap_hac = aggregate_to_substations(n, buses_i) + n, busmap_hac = aggregate_to_substations(n, aggregation_strategies, buses_i) busmaps.append(busmap_hac) if snakemake.wildcards.simpl: From f1dedd9429fe6a5633fdfeead552abfacea48e39 Mon Sep 17 00:00:00 2001 From: martacki Date: Mon, 27 Jun 2022 21:11:49 +0200 Subject: [PATCH 75/90] hierarchical clustering: release notes & revert to old default clustering algorithm (kmeans) --- config.default.yaml | 8 ++++---- config.tutorial.yaml | 8 ++++---- doc/release_notes.rst | 2 ++ test/config.test1.yaml | 8 ++++---- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index e699c38f..e1ac6237 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -22,11 +22,11 @@ countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'F clustering: simplify_network: to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) - algorithm: hac - feature: solar+onwind-time - cluster_network: - algorithm: hac # choose from: [hac, kmeans] + algorithm: kmeans # choose from: [hac, kmeans] feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc. + cluster_network: + algorithm: kmeans + feature: solar+onwind-time aggregation_strategies: generators: p_nom_max: sum # use "min" for more conservative assumptions diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 75703b8a..db20f1a4 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -22,11 +22,11 @@ countries: ['BE'] clustering: simplify_network: to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) - algorithm: hac - feature: solar+onwind-time - cluster_network: - algorithm: hac # choose from: [hac, kmeans] + algorithm: kmeans # choose from: [hac, kmeans] feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc. + cluster_network: + algorithm: kmeans + feature: solar+onwind-time aggregation_strategies: generators: p_nom_max: sum # use "min" for more conservative assumptions diff --git a/doc/release_notes.rst b/doc/release_notes.rst index c000a046..47a67970 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -76,6 +76,8 @@ Upcoming Release * Clustering strategies for generators and buses have moved from distinct scripts to configurables to unify the process and make it more transparent. +* Hierarchical clustering was introduced. Distance metric is calculated from renewable potentials on hourly (feature entry ends with `-time`) or annual (feature entry in config end with `-cap`) values. + PyPSA-Eur 0.4.0 (22th September 2021) ===================================== diff --git a/test/config.test1.yaml b/test/config.test1.yaml index d8d095b8..5aecfdf2 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -21,11 +21,11 @@ countries: ['BE'] clustering: simplify_network: to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) - algorithm: hac - feature: solar+onwind-time - cluster_network: - algorithm: hac # choose from: [hac, kmeans] + algorithm: kmeans # choose from: [hac, kmeans] feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc. + cluster_network: + algorithm: kmeans + feature: solar+onwind-time aggregation_strategies: generators: p_nom_max: sum # use "min" for more conservative assumptions From 9d997fbd790321dee611cd0dbb49683e6ce1cd53 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 28 Jun 2022 10:14:26 +0200 Subject: [PATCH 76/90] generalize conventional attr handling through config --- Snakefile | 3 ++- config.default.yaml | 2 +- ...{nuclear_eafs.csv => nuclear_p_max_pu.csv} | 0 doc/configuration.rst | 13 ++++++++++- doc/release_notes.rst | 6 ++--- scripts/add_electricity.py | 23 +++++++++++-------- 6 files changed, 31 insertions(+), 16 deletions(-) rename data/{nuclear_eafs.csv => nuclear_p_max_pu.csv} (100%) diff --git a/Snakefile b/Snakefile index af2e6e90..22b6107d 100644 --- a/Snakefile +++ b/Snakefile @@ -217,7 +217,8 @@ rule add_electricity: load='resources/load.csv', nuts3_shapes='resources/nuts3_shapes.geojson', **{f"profile_{tech}": f"resources/profile_{tech}.nc" - for tech in config['renewable']} + for tech in config['renewable']}, + **{"conventional_{carrier}_{attrs}": fn for carrier in config.get('conventional', {None: {}}).values() for fn in carrier.values() if str(fn).startswith("data/")}, output: "networks/elec.nc" log: "logs/add_electricity.log" benchmark: "benchmarks/add_electricity" diff --git a/config.default.yaml b/config.default.yaml index 177c5e74..a67562c6 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -187,7 +187,7 @@ renewable: conventional: nuclear: - energy_availability_factors: "data/nuclear-eafs.csv" # float of file name + p_max_pu: "data/nuclear_p_max_pu.csv" # float of file name lines: types: diff --git a/data/nuclear_eafs.csv b/data/nuclear_p_max_pu.csv similarity index 100% rename from data/nuclear_eafs.csv rename to data/nuclear_p_max_pu.csv diff --git a/doc/configuration.rst b/doc/configuration.rst index 67d25228..c332ea7d 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -171,7 +171,7 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia .. literalinclude:: ../config.default.yaml :language: yaml :start-at: hydro: - :end-before: lines: + :end-before: conventional: .. csv-table:: :header-rows: 1 @@ -180,6 +180,17 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia .. _lines_cf: +``conventional`` +============= + +Define additional generator attribute for conventional carrier types. If a scalar value is given it is applied to all generators. However if a string starting with "data/" is given, the value is interpreted as a path to a csv file with country specific values. Then, the values are read in and applied to all generators of the given carrier in the given country. Note that the value(s) overwrite the existing values in the corresponding section of the ``generators`` dataframe. + +.. literalinclude:: ../config.default.yaml + :language: yaml + :start-at: conventional: + :end-before: lines: + + ``lines`` ============= diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 57c0a2f0..3addf3ab 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -53,11 +53,11 @@ Upcoming Release * Add function to add global constraint on use of gas in :mod:`prepare_network`. This can be activated by including the keyword ``CH4L`` in the ``{opts}`` wildcard which enforces the limit set in ``electricity: gaslimit:`` given in MWh thermal. Alternatively, it is possible to append a number in the `{opts}` wildcard, e.g. `CH4L200` which limits the gas use to 200 TWh thermal. -* Add configuration option to implement Energy Availability Factors (EAFs) for conventional generation technologies. - * A new section ``conventional`` was added to the config file. This section contains configurations for conventional carriers. -* Implement country-specific EAFs for nuclear power plants based on IAEA 2018-2020 reported country averages. These are specified ``data/nuclear_eafs.csv`` and translate to static ``p_max_pu`` values. +* Add configuration option to implement arbitrary generator attributes for conventional generation technologies. + +* Implement country-specific Energy Availability Factors (EAFs) for nuclear power plants based on IAEA 2018-2020 reported country averages. These are specified ``data/nuclear_p_max_pu.csv`` and translate to static ``p_max_pu`` values. * The powerplants that have been shut down before 2021 are filtered out. diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index ea95fd94..88c2ce66 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -331,17 +331,20 @@ def attach_conventional_generators(n, costs, ppl, conventional_carriers, extenda # Generators with technology affected idx = n.generators.query("carrier == @carrier").index - factors = conventional_config[carrier].get("energy_availability_factors") - if isinstance(factors, float): - # Single value affecting all generators of technology k indiscriminantely of country - n.generators.loc[idx, "p_max_pu"] = factors - elif isinstance(factors, str): - factors = pd.read_csv(factors, index_col=0)['factor'] - # Values affecting generators of technology k country-specific - # First map generator buses to countries; then map countries to p_max_pu - bus_factors = n.buses.country.map(factors) - n.generators.p_max_pu.update(n.generators.loc[idx].bus.map(bus_factors).dropna()) + for key in list(set(conventional_carriers[carrier]) & set(n.generators)): + + values = conventional_config[carrier][key] + + if isinstance(values, str) and str(values).startswith("data/"): + # Values affecting generators of technology k country-specific + # First map generator buses to countries; then map countries to p_max_pu + values = pd.read_csv(values, index_col=0).iloc[:, 0] + bus_values = n.buses.country.map(values) + n.generators[key].update(n.generators.loc[idx].bus.map(bus_values).dropna()) + else: + # Single value affecting all generators of technology k indiscriminantely of country + n.generators.loc[idx, key] = values From 917f41ef21d97635a1ff1aa8ebca2d7ca0a1255d Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Tue, 28 Jun 2022 13:11:08 +0200 Subject: [PATCH 77/90] Update Snakefile Co-authored-by: Fabian Neumann --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index 22b6107d..c786efdc 100644 --- a/Snakefile +++ b/Snakefile @@ -218,7 +218,7 @@ rule add_electricity: nuts3_shapes='resources/nuts3_shapes.geojson', **{f"profile_{tech}": f"resources/profile_{tech}.nc" for tech in config['renewable']}, - **{"conventional_{carrier}_{attrs}": fn for carrier in config.get('conventional', {None: {}}).values() for fn in carrier.values() if str(fn).startswith("data/")}, + **{f"conventional_{carrier}_{attrs}": fn for carrier in config.get('conventional', {None: {}}).values() for attrs, fn in carrier.items() if str(fn).startswith("data/")}, output: "networks/elec.nc" log: "logs/add_electricity.log" benchmark: "benchmarks/add_electricity" From 3294ad92ebafaa5ace5b3fcbdf2f3d29efe2537f Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Tue, 28 Jun 2022 13:14:47 +0200 Subject: [PATCH 78/90] Update scripts/solve_network.py Co-authored-by: Martha Frysztacki --- scripts/solve_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 6c40ca4f..b3280a94 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -104,7 +104,7 @@ def prepare_network(n, solve_opts): if load_shedding: n.add("Carrier", "load", color="#dd2e23", nice_name="Load shedding") buses_i = n.buses.query("carrier == 'AC'").index - if not np.isscalar(load_shedding): load_shedding = 1e2 + if not np.isscalar(load_shedding): load_shedding = 1e2 # Eur/kWh # intersect between macroeconomic and surveybased # willingness to pay # http://journal.frontiersin.org/article/10.3389/fenrg.2015.00055/full) From 522f218eed4cacd8d2d37ea3b034ca6b38e60c52 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 28 Jun 2022 13:15:37 +0200 Subject: [PATCH 79/90] Snakefile: rename attrs to attr in add_electricity input function --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index c786efdc..9e8d6236 100644 --- a/Snakefile +++ b/Snakefile @@ -218,7 +218,7 @@ rule add_electricity: nuts3_shapes='resources/nuts3_shapes.geojson', **{f"profile_{tech}": f"resources/profile_{tech}.nc" for tech in config['renewable']}, - **{f"conventional_{carrier}_{attrs}": fn for carrier in config.get('conventional', {None: {}}).values() for attrs, fn in carrier.items() if str(fn).startswith("data/")}, + **{f"conventional_{carrier}_{attr}": fn for carrier in config.get('conventional', {None: {}}).values() for attr, fn in carrier.items() if str(fn).startswith("data/")}, output: "networks/elec.nc" log: "logs/add_electricity.log" benchmark: "benchmarks/add_electricity" From 82013fd0816c380e148e4b214408254d84f8c946 Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Tue, 28 Jun 2022 13:16:55 +0200 Subject: [PATCH 80/90] Update scripts/add_electricity.py Co-authored-by: Martha Frysztacki --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 88c2ce66..d244a381 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -332,7 +332,7 @@ def attach_conventional_generators(n, costs, ppl, conventional_carriers, extenda # Generators with technology affected idx = n.generators.query("carrier == @carrier").index - for key in list(set(conventional_carriers[carrier]) & set(n.generators)): + for key in list(set(conventional_config[carrier]) & set(n.generators)): values = conventional_config[carrier][key] From e9e00291c501b473146d12098510be3e4e96eccc Mon Sep 17 00:00:00 2001 From: martacki Date: Tue, 28 Jun 2022 13:40:14 +0200 Subject: [PATCH 81/90] update configtables/opts to include CH4L option --- doc/configtables/opts.csv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/configtables/opts.csv b/doc/configtables/opts.csv index 918d0d17..6a72dd01 100644 --- a/doc/configtables/opts.csv +++ b/doc/configtables/opts.csv @@ -8,4 +8,5 @@ Trigger, Description, Definition, Status ``ATK``, "Require each node to be autarkic. Example: ``ATK`` removes all lines and links. ``ATKc`` removes all cross-border lines and links.", ``prepare_network``, In active use ``BAU``, Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities``, ``solve_network``: `add_opts_constraints() `__, Untested ``SAFE``, Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network., ``solve_network`` `add_opts_constraints() `__, Untested -``carrier+{c|p}factor``, "Alter the capital cost (``c``) or installable potential (``p``) of a carrier by a factor. Example: ``solar+c0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use +``carrier+{c|p|m}factor``,"Alter the capital cost (``c``), installable potential (``p``) or marginal costs (``m``) of a carrier by a factor. Example: ``solar+c0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use +``CH4L``,"Add an overall absolute gas limit. If configured in ``electricity: gaslimit`` it is given in MWh thermal, if a float is appended, the overall gaslimit is assumed to be given in TWh thermal (e.g. ``CH4L200`` limits gas dispatch to 200 TWh termal)", ``prepare_network``: ``add_gaslimit()``, In active use \ No newline at end of file From 67ac464b6a0b66e012b1dcaae9748c0c5722e0a6 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 28 Jun 2022 16:33:46 +0200 Subject: [PATCH 82/90] add_electricity: use conventional_inputs from snakemake.input for attach_conventional_generators --- Snakefile | 2 +- scripts/add_electricity.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Snakefile b/Snakefile index a8d98057..576d1fbb 100644 --- a/Snakefile +++ b/Snakefile @@ -229,7 +229,7 @@ rule add_electricity: nuts3_shapes='resources/nuts3_shapes.geojson', **{f"profile_{tech}": f"resources/profile_{tech}.nc" for tech in config['renewable']}, - **{f"conventional_{carrier}_{attr}": fn for carrier in config.get('conventional', {None: {}}).values() for attr, fn in carrier.items() if str(fn).startswith("data/")}, + **{f"conventional_{carrier}_{attr}": fn for carrier, d in config.get('conventional', {None: {}}).items() for attr, fn in d.items() if str(fn).startswith("data/")}, output: "networks/elec.nc" log: "logs/add_electricity.log" benchmark: "benchmarks/add_electricity" diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index d244a381..342b12e9 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -302,7 +302,7 @@ def attach_wind_and_solar(n, costs, input_profiles, technologies, extendable_car p_max_pu=ds['profile'].transpose('time', 'bus').to_pandas()) -def attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers, conventional_config): +def attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers, conventional_config, conventional_inputs): carriers = set(conventional_carriers) | set(extendable_carriers['Generator']) _add_missing_carriers_from_costs(n, costs, carriers) @@ -332,19 +332,19 @@ def attach_conventional_generators(n, costs, ppl, conventional_carriers, extenda # Generators with technology affected idx = n.generators.query("carrier == @carrier").index - for key in list(set(conventional_config[carrier]) & set(n.generators)): + for attr in list(set(conventional_config[carrier]) & set(n.generators)): - values = conventional_config[carrier][key] + values = conventional_config[carrier][attr] - if isinstance(values, str) and str(values).startswith("data/"): + if f"conventional_{carrier}_{attr}" in conventional_inputs: # Values affecting generators of technology k country-specific # First map generator buses to countries; then map countries to p_max_pu values = pd.read_csv(values, index_col=0).iloc[:, 0] bus_values = n.buses.country.map(values) - n.generators[key].update(n.generators.loc[idx].bus.map(bus_values).dropna()) + n.generators[attr].update(n.generators.loc[idx].bus.map(bus_values).dropna()) else: # Single value affecting all generators of technology k indiscriminantely of country - n.generators.loc[idx, key] = values + n.generators.loc[idx, attr] = values @@ -603,7 +603,8 @@ if __name__ == "__main__": update_transmission_costs(n, costs, snakemake.config['lines']['length_factor']) - attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers, snakemake.config.get("conventional", {})) + conventional_inputs = {k: v for k, v in snakemake.input.items() if k.startswith("conventional_")} + attach_conventional_generators(n, costs, ppl, conventional_carriers, extendable_carriers, snakemake.config.get("conventional", {}), conventional_inputs) attach_wind_and_solar(n, costs, snakemake.input, renewable_carriers, extendable_carriers, snakemake.config['lines']['length_factor']) From 27e5dec4e69277322b758311bc7a251272faf145 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 30 Jun 2022 09:01:49 +0200 Subject: [PATCH 83/90] documentation fixes --- config.default.yaml | 38 +++++++++++++++++----------------- config.tutorial.yaml | 38 +++++++++++++++++----------------- doc/configtables/snapshots.csv | 2 +- doc/installation.rst | 2 ++ doc/tutorial.rst | 15 +++++++------- scripts/retrieve_databundle.py | 2 +- test/config.test1.yaml | 38 +++++++++++++++++----------------- 7 files changed, 69 insertions(+), 66 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index 173feb51..d185b5d1 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -17,25 +17,6 @@ scenario: countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'] -clustering: - simplify_network: - to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) - algorithm: kmeans # choose from: [hac, kmeans] - feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc. - cluster_network: - algorithm: kmeans - feature: solar+onwind-time - aggregation_strategies: - generators: - p_nom_max: sum # use "min" for more conservative assumptions - p_nom_min: sum - p_min_pu: mean - marginal_cost: mean - committable: any - ramp_limit_up: max - ramp_limit_down: max - efficiency: mean - snapshots: start: "2013-01-01" end: "2014-01-01" @@ -251,6 +232,25 @@ costs: emission_prices: # in currency per tonne emission, only used with the option Ep co2: 0. +clustering: + simplify_network: + to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + algorithm: kmeans # choose from: [hac, kmeans] + feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc. + cluster_network: + algorithm: kmeans + feature: solar+onwind-time + aggregation_strategies: + generators: + p_nom_max: sum # use "min" for more conservative assumptions + p_nom_min: sum + p_min_pu: mean + marginal_cost: mean + committable: any + ramp_limit_up: max + ramp_limit_down: max + efficiency: mean + solving: options: formulation: kirchhoff diff --git a/config.tutorial.yaml b/config.tutorial.yaml index b49f24ba..00b8576a 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -18,25 +18,6 @@ scenario: countries: ['BE'] -clustering: - simplify_network: - to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) - algorithm: kmeans # choose from: [hac, kmeans] - feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc. - cluster_network: - algorithm: kmeans - feature: solar+onwind-time - aggregation_strategies: - generators: - p_nom_max: sum # use "min" for more conservative assumptions - p_nom_min: sum - p_min_pu: mean - marginal_cost: mean - committable: any - ramp_limit_up: max - ramp_limit_down: max - efficiency: mean - snapshots: start: "2013-03-01" end: "2013-04-01" @@ -184,6 +165,25 @@ costs: emission_prices: # in currency per tonne emission, only used with the option Ep co2: 0. +clustering: + simplify_network: + to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + algorithm: kmeans # choose from: [hac, kmeans] + feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc. + cluster_network: + algorithm: kmeans + feature: solar+onwind-time + aggregation_strategies: + generators: + p_nom_max: sum # use "min" for more conservative assumptions + p_nom_min: sum + p_min_pu: mean + marginal_cost: mean + committable: any + ramp_limit_up: max + ramp_limit_down: max + efficiency: mean + solving: options: formulation: kirchhoff diff --git a/doc/configtables/snapshots.csv b/doc/configtables/snapshots.csv index 4d917f4d..00297498 100644 --- a/doc/configtables/snapshots.csv +++ b/doc/configtables/snapshots.csv @@ -1,4 +1,4 @@ ,Unit,Values,Description start,--,"str or datetime-like; e.g. YYYY-MM-DD","Left bound of date range" end,--,"str or datetime-like; e.g. YYYY-MM-DD","Right bound of date range" -closed,--,"One of {None, ‘left’, ‘right’}","Make the time interval closed to the ``left``, ``right``, or both sides ``None``." +closed,--,"One of {None, ‘left’, ‘right’}","Make the time interval closed to the ``left``, ``right``, or open on both sides ``None``." diff --git a/doc/installation.rst b/doc/installation.rst index 16fdf766..aea25a42 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -102,6 +102,8 @@ It might be the case that you can only retrieve solutions by using a commercial conda activate pypsa-eur conda install -c conda-forge ipopt glpk +.. warning:: + On Windows, new versions of ``ipopt`` have caused problems. Consider downgrading to version 3.11.1. .. _defaultconfig: diff --git a/doc/tutorial.rst b/doc/tutorial.rst index c37abb39..93143783 100644 --- a/doc/tutorial.rst +++ b/doc/tutorial.rst @@ -47,7 +47,8 @@ The model can be adapted to only include selected countries (e.g. Belgium) inste .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 20 + :start-at: countries: + :end-before: snapshots: Likewise, the example's temporal scope can be restricted (e.g. to a single month). @@ -60,14 +61,14 @@ It is also possible to allow less or more carbon-dioxide emissions. Here, we lim .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 40,42 + :lines: 35,37 PyPSA-Eur also includes a database of existing conventional powerplants. -We can select which types of powerplants we like to be included with fixed capacities: +We can select which types of powerplants we like to be included: .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 40,56 + :lines: 35,51 To accurately model the temporal and spatial availability of renewables such as wind and solar energy, we rely on historical weather data. It is advisable to adapt the required range of coordinates to the selection of countries. @@ -82,14 +83,14 @@ For example, we may want to use the ERA-5 dataset for solar and not the default .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 67,110,111 + :lines: 62,105,106 Finally, it is possible to pick a solver. For instance, this tutorial uses the open-source solvers CBC and Ipopt and does not rely on the commercial solvers Gurobi or CPLEX (for which free academic licenses are available). .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 173,183,184 + :lines: 187,197,198 .. note:: @@ -284,4 +285,4 @@ The solved networks can be analysed just like any other PyPSA network (e.g. in J network = pypsa.Network("results/networks/elec_s_6_ec_lcopt_Co2L-24H.nc") -For inspiration, read the `examples section in the PyPSA documentation `_. +For inspiration, read the `examples section in the PyPSA documentation `_. diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index 86869879..d29ce32b 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -11,7 +11,7 @@ The data bundle (1.4 GB) contains common GIS datasets like NUTS3 shapes, EEZ sha This rule downloads the data bundle from `zenodo `_ and extracts it in the ``data`` sub-directory, such that all files of the bundle are stored in the ``data/bundle`` subdirectory. -The :ref:`tutorial` uses a smaller `data bundle `_ than required for the full model (19 MB) +The :ref:`tutorial` uses a smaller `data bundle `_ than required for the full model (188 MB) .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3517921.svg :target: https://doi.org/10.5281/zenodo.3517921 diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 66e65a7c..b3f63fa8 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -17,25 +17,6 @@ scenario: countries: ['BE'] -clustering: - simplify_network: - to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) - algorithm: kmeans # choose from: [hac, kmeans] - feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc. - cluster_network: - algorithm: kmeans - feature: solar+onwind-time - aggregation_strategies: - generators: - p_nom_max: sum # use "min" for more conservative assumptions - p_nom_min: sum - p_min_pu: mean - marginal_cost: mean - committable: any - ramp_limit_up: max - ramp_limit_down: max - efficiency: mean - snapshots: start: "2013-03-01" end: "2013-03-08" @@ -182,6 +163,25 @@ costs: emission_prices: # only used with the option Ep co2: 0. +clustering: + simplify_network: + to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + algorithm: kmeans # choose from: [hac, kmeans] + feature: solar+onwind-time # only for hac. choose from: [solar+onwind-time, solar+onwind-cap, solar-time, solar-cap, solar+offwind-cap] etc. + cluster_network: + algorithm: kmeans + feature: solar+onwind-time + aggregation_strategies: + generators: + p_nom_max: sum # use "min" for more conservative assumptions + p_nom_min: sum + p_min_pu: mean + marginal_cost: mean + committable: any + ramp_limit_up: max + ramp_limit_down: max + efficiency: mean + solving: options: formulation: kirchhoff From 44f793be91c006b7eb6ffcf8ef91dd232026b8c5 Mon Sep 17 00:00:00 2001 From: Julio Pascual <108464786+jpscl@users.noreply.github.com> Date: Mon, 4 Jul 2022 11:49:55 +0200 Subject: [PATCH 84/90] Changed doc info in retrieve_databundle.py Simply changed the text regarding the output directory. It said 'cutouts/bundle' but I think it should be 'data/bundle'. It only affects the documentation. --- scripts/retrieve_databundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index d29ce32b..5f05c575 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -28,7 +28,7 @@ The :ref:`tutorial` uses a smaller `data bundle Date: Mon, 4 Jul 2022 18:41:08 +0200 Subject: [PATCH 85/90] Update build_load_data.py --- scripts/build_load_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_load_data.py b/scripts/build_load_data.py index 55270e49..13dc1363 100755 --- a/scripts/build_load_data.py +++ b/scripts/build_load_data.py @@ -30,7 +30,7 @@ Inputs Outputs ------- -- ``resource/time_series_60min_singleindex_filtered.csv``: +- ``resources/load.csv``: """ From ce3b193a88ad7290ed8b669df4d9a550e9b80978 Mon Sep 17 00:00:00 2001 From: Julio Pascual <108464786+jpscl@users.noreply.github.com> Date: Mon, 4 Jul 2022 18:42:22 +0200 Subject: [PATCH 86/90] Update add_electricity.py --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 342b12e9..0fb025df 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -53,7 +53,7 @@ Inputs :scale: 34 % - ``data/geth2015_hydro_capacities.csv``: alternative to capacities above; not currently used! -- ``resources/opsd_load.csv`` Hourly per-country load profiles. +- ``resources/load.csv`` Hourly per-country load profiles. - ``resources/regions_onshore.geojson``: confer :ref:`busregions` - ``resources/nuts3_shapes.geojson``: confer :ref:`shapes` - ``resources/powerplants.csv``: confer :ref:`powerplants` From 433f0af4093107b49093d388ea8416306d59e010 Mon Sep 17 00:00:00 2001 From: Julio Pascual <108464786+jpscl@users.noreply.github.com> Date: Mon, 4 Jul 2022 18:51:49 +0200 Subject: [PATCH 87/90] Update build_load_data.py --- scripts/build_load_data.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build_load_data.py b/scripts/build_load_data.py index 13dc1363..b0317f5e 100755 --- a/scripts/build_load_data.py +++ b/scripts/build_load_data.py @@ -26,6 +26,8 @@ Relevant Settings Inputs ------ +- ``data/load_raw.csv``: + Outputs ------- From 5d98cc7e97996283580462d1a50669be5eb81947 Mon Sep 17 00:00:00 2001 From: Julio Pascual <108464786+jpscl@users.noreply.github.com> Date: Mon, 4 Jul 2022 18:53:41 +0200 Subject: [PATCH 88/90] Update build_load_data.py --- scripts/build_load_data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/build_load_data.py b/scripts/build_load_data.py index b0317f5e..ac6de2b1 100755 --- a/scripts/build_load_data.py +++ b/scripts/build_load_data.py @@ -28,7 +28,6 @@ Inputs - ``data/load_raw.csv``: - Outputs ------- From a860643dd8cd66418b6a6d7893960c636d78ec3b Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 18 Jul 2022 17:28:09 +0200 Subject: [PATCH 89/90] reduce number of columns for simplified and clustered regions --- scripts/cluster_network.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index bd412b7f..1ef2f2e5 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -122,7 +122,7 @@ Exemplary unsolved network clustered to 37 nodes: """ import logging -from _helpers import configure_logging, update_p_nom_max, get_aggregation_strategies, REGION_COLS +from _helpers import configure_logging, update_p_nom_max, get_aggregation_strategies import pypsa import os @@ -372,9 +372,8 @@ def cluster_regions(busmaps, input=None, output=None): for which in ('regions_onshore', 'regions_offshore'): regions = gpd.read_file(getattr(input, which)) - regions = regions.reindex(columns=REGION_COLS).set_index('name') - aggfunc = dict(x="mean", y="mean", country="first") - regions_c = regions.dissolve(busmap, aggfunc=aggfunc) + regions = regions.reindex(columns=["name", "geometry"]).set_index('name') + regions_c = regions.dissolve(busmap) regions_c.index.name = 'name' regions_c = regions_c.reset_index() regions_c.to_file(getattr(output, which)) From 6047c8d7508bcc7e24b018adda2ce90c9d1682ac Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 20 Jul 2022 11:47:07 +0200 Subject: [PATCH 90/90] add mergedeep to dependencies --- envs/environment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/envs/environment.yaml b/envs/environment.yaml index 4aefcb9a..8bd7428f 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -53,6 +53,7 @@ dependencies: - tqdm - pytz - tabula-py + - mergedeep - pip: - vresutils>=0.3.1