From bef4967e84567434aa836d6d572c0316abe6f96f Mon Sep 17 00:00:00 2001 From: martacki Date: Mon, 20 Jun 2022 18:58:23 +0200 Subject: [PATCH 1/7] clustering strategies moved to configurables --- config.default.yaml | 4 ++++ config.tutorial.yaml | 4 ++++ scripts/cluster_network.py | 43 +++++++++++++++-------------------- scripts/simplify_network.py | 45 ++++++++++++++++++++----------------- 4 files changed, 51 insertions(+), 45 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index d2bf6159..9462719b 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -22,6 +22,10 @@ countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'F clustering: simplify: to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + aggregation_strategies: + generators: + p_nom_max: "sum" # use "min" for more conservative assumptions + p_nom_min: "sum" snapshots: start: "2013-01-01" diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 225d8f78..4dc7a94d 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -22,6 +22,10 @@ countries: ['BE'] clustering: simplify: to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + aggregation_strategies: + generators: + p_nom_max: "sum" # use "min" for more conservative assumptions + p_nom_min: "sum" snapshots: start: "2013-03-01" diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 1d5608e2..0a3d768a 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -11,11 +11,10 @@ Relevant Settings .. code:: yaml - focus_weights: + clustering: + aggregation_strategies: - renewable: (keys) - {technology}: - potential: + focus_weights: solving: solver: @@ -259,15 +258,16 @@ def busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights=None, algori def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carriers=None, - line_length_factor=1.25, potential_mode='simple', solver_name="cbc", + line_length_factor=1.25, aggregation_strategies=dict(), solver_name="cbc", algorithm="kmeans", extended_link_costs=0, focus_weights=None): - if potential_mode == 'simple': - p_nom_max_strategy = pd.Series.sum - elif potential_mode == 'conservative': - p_nom_max_strategy = pd.Series.min - else: - raise AttributeError(f"potential_mode should be one of 'simple' or 'conservative' but is '{potential_mode}'") + bus_strategies = dict(country=_make_consense("Bus", "country")) + bus_strategies.update(aggregation_strategies.get("buses", {})) + generator_strategies = aggregation_strategies.get("generators", {"p_nom_max": "sum"}) + + # this snippet supports compatibility of PyPSA and PyPSA-EUR: + if "p_nom_max" in generator_strategies: + if generator_strategies["p_nom_max"] == "min": generator_strategies["p_nom_max"] = np.min if not isinstance(custom_busmap, pd.Series): busmap = busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights, algorithm) @@ -276,19 +276,12 @@ def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carr clustering = get_clustering_from_busmap( n, busmap, - bus_strategies=dict(country=_make_consense("Bus", "country")), + bus_strategies=bus_strategies, aggregate_generators_weighted=True, aggregate_generators_carriers=aggregate_carriers, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=line_length_factor, - generator_strategies={'p_nom_max': p_nom_max_strategy, - 'p_nom_min': pd.Series.sum, - 'p_min_pu': pd.Series.mean, - 'marginal_cost': pd.Series.mean, - 'committable': np.any, - 'ramp_limit_up': pd.Series.max, - 'ramp_limit_down': pd.Series.max, - }, + generator_strategies=generator_strategies, scale_link_capital_costs=False) if not n.links.empty: @@ -375,8 +368,8 @@ if __name__ == "__main__": "The `potential` configuration option must agree for all renewable carriers, for now!" ) return v - potential_mode = consense(pd.Series([snakemake.config['renewable'][tech]['potential'] - for tech in renewable_carriers])) + aggregation_strategies = snakemake.config["clustering"].get("aggregation_strategies", {}) + custom_busmap = snakemake.config["enable"].get("custom_busmap", False) if custom_busmap: custom_busmap = pd.read_csv(snakemake.input.custom_busmap, index_col=0, squeeze=True) @@ -384,12 +377,12 @@ if __name__ == "__main__": logger.info(f"Imported custom busmap from {snakemake.input.custom_busmap}") clustering = clustering_for_n_clusters(n, n_clusters, custom_busmap, aggregate_carriers, - line_length_factor, potential_mode, + line_length_factor, aggregation_strategies, snakemake.config['solving']['solver']['name'], "kmeans", hvac_overhead_cost, focus_weights) - - update_p_nom_max(n) + update_p_nom_max(clustering.network) + clustering.network.export_to_netcdf(snakemake.output.network) for attr in ('busmap', 'linemap'): #also available: linemap_positive, linemap_negative getattr(clustering, attr).to_csv(snakemake.output[attr]) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 287dfe32..7d51c511 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -13,6 +13,10 @@ Relevant Settings .. code:: yaml + clustering: + simplify: + aggregation_strategies: + costs: USD2013_to_EUR2013: discountrate: @@ -22,10 +26,6 @@ Relevant Settings electricity: max_hours: - renewables: (keys) - {technology}: - potential: - lines: length_factor: @@ -320,7 +320,7 @@ def remove_stubs(n, costs, config, output): return n, busmap -def aggregate_to_substations(n, buses_i=None): +def aggregate_to_substations(n, config, aggregation_strategies=dict(), buses_i=None): # can be used to aggregate a selection of buses to electrically closest neighbors # if no buses are given, nodes that are no substations or without offshore connection are aggregated @@ -345,19 +345,29 @@ def aggregate_to_substations(n, buses_i=None): busmap = n.buses.index.to_series() busmap.loc[buses_i] = dist.idxmin(1) + # default aggregation strategies must be specified within the function, otherwise (when defaults are passed in + # the function's definition) they get lost in case custom values for different variables are specified in the config + bus_strategies = dict(country=_make_consense("Bus", "country")) + bus_strategies.update(aggregation_strategies.get("buses", {})) + generator_strategies = aggregation_strategies.get("generators", {"p_nom_max": "sum"}) + + # this snippet supports compatibility of PyPSA and PyPSA-EUR: + if "p_nom_max" in generator_strategies: + if generator_strategies["p_nom_max"] == "min": generator_strategies["p_nom_max"] = np.min + clustering = get_clustering_from_busmap(n, busmap, - bus_strategies=dict(country=_make_consense("Bus", "country")), + bus_strategies=bus_strategies, aggregate_generators_weighted=True, aggregate_generators_carriers=None, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=1.0, - generator_strategies={'p_nom_max': 'sum'}, + generator_strategies=generator_strategies, scale_link_capital_costs=False) return clustering.network, busmap -def cluster(n, n_clusters, config): +def cluster(n, n_clusters, config, aggregation_strategies=dict()): logger.info(f"Clustering to {n_clusters} buses") focus_weights = config.get('focus_weights', None) @@ -365,16 +375,9 @@ def cluster(n, n_clusters, config): renewable_carriers = pd.Index([tech for tech in n.generators.carrier.unique() if tech.split('-', 2)[0] in config['renewable']]) - def consense(x): - v = x.iat[0] - assert ((x == v).all() or x.isnull().all()), ( - "The `potential` configuration option must agree for all renewable carriers, for now!" - ) - return v - potential_mode = (consense(pd.Series([config['renewable'][tech]['potential'] - for tech in renewable_carriers])) - if len(renewable_carriers) > 0 else 'conservative') - clustering = clustering_for_n_clusters(n, n_clusters, custom_busmap=False, potential_mode=potential_mode, + + clustering = clustering_for_n_clusters(n, n_clusters, custom_busmap=False, + aggregation_strategies=aggregation_strategies, solver_name=config['solving']['solver']['name'], focus_weights=focus_weights) @@ -389,6 +392,8 @@ if __name__ == "__main__": n = pypsa.Network(snakemake.input.network) + aggregation_strategies = snakemake.config["clustering"].get("aggregation_strategies", {}) + n, trafo_map = simplify_network_to_380(n) Nyears = n.snapshot_weightings.objective.sum() / 8760 @@ -402,11 +407,11 @@ if __name__ == "__main__": busmaps = [trafo_map, simplify_links_map, stub_map] if snakemake.config.get('clustering', {}).get('simplify', {}).get('to_substations', False): - n, substation_map = aggregate_to_substations(n) + n, substation_map = aggregate_to_substations(n, snakemake.config, aggregation_strategies) busmaps.append(substation_map) if snakemake.wildcards.simpl: - n, cluster_map = cluster(n, int(snakemake.wildcards.simpl), snakemake.config) + n, cluster_map = cluster(n, int(snakemake.wildcards.simpl), snakemake.config, aggregation_strategies) busmaps.append(cluster_map) # some entries in n.buses are not updated in previous functions, therefore can be wrong. as they are not needed From b4d8dd8ecb827c3ed450375e80fcf3717cc2aaab Mon Sep 17 00:00:00 2001 From: martacki Date: Mon, 20 Jun 2022 19:21:08 +0200 Subject: [PATCH 2/7] add changes from PR #379 --- config.default.yaml | 9 +++++++-- config.tutorial.yaml | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index 9462719b..7162d449 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -24,8 +24,13 @@ clustering: to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) aggregation_strategies: generators: - p_nom_max: "sum" # use "min" for more conservative assumptions - p_nom_min: "sum" + p_nom_max: sum # use "min" for more conservative assumptions + p_nom_min: sum + p_min_pu: mean + marginal_cost: mean + committable: any + ramp_limit_up: max + ramp_limit_down: max snapshots: start: "2013-01-01" diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 4dc7a94d..f18f23f4 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -24,8 +24,13 @@ clustering: to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) aggregation_strategies: generators: - p_nom_max: "sum" # use "min" for more conservative assumptions - p_nom_min: "sum" + p_nom_max: sum # use "min" for more conservative assumptions + p_nom_min: sum + p_min_pu: mean + marginal_cost: mean + committable: any + ramp_limit_up: max + ramp_limit_down: max snapshots: start: "2013-03-01" From bdd0cc3aa125fa5a721357ebfc47418b79475716 Mon Sep 17 00:00:00 2001 From: martacki Date: Tue, 21 Jun 2022 18:42:49 +0200 Subject: [PATCH 3/7] clustering strats to configurables: review suggestions --- scripts/cluster_network.py | 11 +++++++---- scripts/simplify_network.py | 19 +++++++++++-------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 0a3d768a..fd66b043 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -263,11 +263,8 @@ def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carr bus_strategies = dict(country=_make_consense("Bus", "country")) bus_strategies.update(aggregation_strategies.get("buses", {})) - generator_strategies = aggregation_strategies.get("generators", {"p_nom_max": "sum"}) - # this snippet supports compatibility of PyPSA and PyPSA-EUR: - if "p_nom_max" in generator_strategies: - if generator_strategies["p_nom_max"] == "min": generator_strategies["p_nom_max"] = np.min + generator_strategies = aggregation_strategies.get("generators", {"p_nom_max": "sum"}) if not isinstance(custom_busmap, pd.Series): busmap = busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights, algorithm) @@ -369,6 +366,12 @@ if __name__ == "__main__": ) return v aggregation_strategies = snakemake.config["clustering"].get("aggregation_strategies", {}) + aggregation_strategies = {} + # translate str entries of aggregation_strategies to pd.Series functions: + aggregation_strategies = { + p: {k: getattr(pd.Series, v) for k,v in aggregation_strategies[p].items()} + for p in aggregation_strategies.keys() + } custom_busmap = snakemake.config["enable"].get("custom_busmap", False) if custom_busmap: diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 7d51c511..52e0c815 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -320,7 +320,7 @@ def remove_stubs(n, costs, config, output): return n, busmap -def aggregate_to_substations(n, config, aggregation_strategies=dict(), buses_i=None): +def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None): # can be used to aggregate a selection of buses to electrically closest neighbors # if no buses are given, nodes that are no substations or without offshore connection are aggregated @@ -345,15 +345,13 @@ def aggregate_to_substations(n, config, aggregation_strategies=dict(), buses_i=N busmap = n.buses.index.to_series() busmap.loc[buses_i] = dist.idxmin(1) - # default aggregation strategies must be specified within the function, otherwise (when defaults are passed in - # the function's definition) they get lost in case custom values for different variables are specified in the config + # default aggregation strategies must be specified within the function, otherwise (when defaults + # are passed in the function's definition) they get lost in case custom values for different + # variables are specified in the config. bus_strategies = dict(country=_make_consense("Bus", "country")) bus_strategies.update(aggregation_strategies.get("buses", {})) - generator_strategies = aggregation_strategies.get("generators", {"p_nom_max": "sum"}) - # this snippet supports compatibility of PyPSA and PyPSA-EUR: - if "p_nom_max" in generator_strategies: - if generator_strategies["p_nom_max"] == "min": generator_strategies["p_nom_max"] = np.min + generator_strategies = aggregation_strategies.get("generators", {"p_nom_max": "sum"}) clustering = get_clustering_from_busmap(n, busmap, bus_strategies=bus_strategies, @@ -393,6 +391,11 @@ if __name__ == "__main__": n = pypsa.Network(snakemake.input.network) aggregation_strategies = snakemake.config["clustering"].get("aggregation_strategies", {}) + # translate str entries of aggregation_strategies to pd.Series functions: + aggregation_strategies = { + p: {k: getattr(pd.Series, v) for k,v in aggregation_strategies[p].items()} + for p in aggregation_strategies.keys() + } n, trafo_map = simplify_network_to_380(n) @@ -407,7 +410,7 @@ if __name__ == "__main__": busmaps = [trafo_map, simplify_links_map, stub_map] if snakemake.config.get('clustering', {}).get('simplify', {}).get('to_substations', False): - n, substation_map = aggregate_to_substations(n, snakemake.config, aggregation_strategies) + n, substation_map = aggregate_to_substations(n, aggregation_strategies) busmaps.append(substation_map) if snakemake.wildcards.simpl: From 8dba48c4125a5578a65f4cf96c94f33780fb36ca Mon Sep 17 00:00:00 2001 From: martacki Date: Tue, 21 Jun 2022 19:08:22 +0200 Subject: [PATCH 4/7] clustering strats to configurables: documentation and testing --- doc/configtables/clustering.csv | 5 +++++ doc/release_notes.rst | 1 + test/config.test1.yaml | 9 +++++++++ 3 files changed, 15 insertions(+) diff --git a/doc/configtables/clustering.csv b/doc/configtables/clustering.csv index 2f63f955..f178ff5c 100644 --- a/doc/configtables/clustering.csv +++ b/doc/configtables/clustering.csv @@ -1,3 +1,8 @@ ,Unit,Values,Description simplify,,, -- to_substations,bool,"{'true','false'}","Aggregates all nodes without power injection (positive or negative, i.e. demand or generation) to electrically closest ones" +-- aggregation_strategies,,, +-- -- generators,,, +-- -- -- {key},str,"{key} can be any of the component of the generator (str). It’s value can be any that can be converted to pandas.Series using getattr(). For example one of {min, max, sum}.","Aggregates the component according to the given strategy. For example, if sum, then all values within each cluster are summed to represent the new generator." +-- -- buses,,, +-- -- -- {key},str,"{key} can be any of the component of the bus (str). It’s value can be any that can be converted to pandas.Series using getattr(). For example one of {min, max, sum}.","Aggregates the component according to the given strategy. For example, if sum, then all values within each cluster are summed to represent the new bus." diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 963a1175..c000a046 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -74,6 +74,7 @@ Upcoming Release * Update rasterio version to correctly calculate exclusion raster +* Clustering strategies for generators and buses have moved from distinct scripts to configurables to unify the process and make it more transparent. PyPSA-Eur 0.4.0 (22th September 2021) ===================================== diff --git a/test/config.test1.yaml b/test/config.test1.yaml index a9ce1e50..e3f39ab5 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -21,6 +21,15 @@ countries: ['BE'] clustering: simplify: to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + aggregation_strategies: + generators: + p_nom_max: sum # use "min" for more conservative assumptions + p_nom_min: sum + p_min_pu: mean + marginal_cost: mean + committable: any + ramp_limit_up: max + ramp_limit_down: max snapshots: start: "2013-03-01" From c9c738e96b2240c8522f84a1941b163069c0e351 Mon Sep 17 00:00:00 2001 From: martacki Date: Fri, 24 Jun 2022 20:57:53 +0200 Subject: [PATCH 5/7] clustering strats to configurables: set defaults for yaml-incompatible strats + include in stubaggregation --- config.default.yaml | 1 + config.tutorial.yaml | 1 + scripts/cluster_network.py | 6 +++--- scripts/simplify_network.py | 37 +++++++++++++++++++++++++------------ test/config.test1.yaml | 1 + 5 files changed, 31 insertions(+), 15 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index 7162d449..144f416e 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -31,6 +31,7 @@ clustering: committable: any ramp_limit_up: max ramp_limit_down: max + efficiency: mean snapshots: start: "2013-01-01" diff --git a/config.tutorial.yaml b/config.tutorial.yaml index f18f23f4..31ca7f99 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -31,6 +31,7 @@ clustering: committable: any ramp_limit_up: max ramp_limit_down: max + efficiency:mean snapshots: start: "2013-03-01" diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index fd66b043..93cd89cd 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -264,7 +264,8 @@ def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carr bus_strategies = dict(country=_make_consense("Bus", "country")) bus_strategies.update(aggregation_strategies.get("buses", {})) - generator_strategies = aggregation_strategies.get("generators", {"p_nom_max": "sum"}) + generator_strategies = {'build_year': lambda x: 0, 'lifetime': lambda x: np.inf} + generator_strategies.update(aggregation_strategies.get("generators", {})) if not isinstance(custom_busmap, pd.Series): busmap = busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights, algorithm) @@ -366,7 +367,6 @@ if __name__ == "__main__": ) return v aggregation_strategies = snakemake.config["clustering"].get("aggregation_strategies", {}) - aggregation_strategies = {} # translate str entries of aggregation_strategies to pd.Series functions: aggregation_strategies = { p: {k: getattr(pd.Series, v) for k,v in aggregation_strategies[p].items()} @@ -383,7 +383,7 @@ if __name__ == "__main__": line_length_factor, aggregation_strategies, snakemake.config['solving']['solver']['name'], "kmeans", hvac_overhead_cost, focus_weights) - + update_p_nom_max(clustering.network) clustering.network.export_to_netcdf(snakemake.output.network) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 52e0c815..0fda5c77 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -189,7 +189,10 @@ def _adjust_capital_costs_using_connection_costs(n, connection_costs_to_bus, out -def _aggregate_and_move_components(n, busmap, connection_costs_to_bus, output, aggregate_one_ports={"Load", "StorageUnit"}): +def _aggregate_and_move_components(n, busmap, connection_costs_to_bus, output, + aggregate_one_ports={"Load", "StorageUnit"}, + aggregation_strategies=dict()): + def replace_components(n, c, df, pnl): n.mremove(c, n.df(c).index) @@ -200,7 +203,12 @@ def _aggregate_and_move_components(n, busmap, connection_costs_to_bus, output, a _adjust_capital_costs_using_connection_costs(n, connection_costs_to_bus, output) - generators, generators_pnl = aggregategenerators(n, busmap, custom_strategies={'p_nom_min': np.sum}) + generator_strategies = {'build_year': lambda x: 0, 'lifetime': lambda x: np.inf} + generator_strategies.update(aggregation_strategies.get("generators", {})) + + generators, generators_pnl = aggregategenerators( + n, busmap, custom_strategies=generator_strategies + ) replace_components(n, "Generator", generators, generators_pnl) for one_port in aggregate_one_ports: @@ -214,7 +222,7 @@ def _aggregate_and_move_components(n, busmap, connection_costs_to_bus, output, a n.mremove(c, df.index[df.bus0.isin(buses_to_del) | df.bus1.isin(buses_to_del)]) -def simplify_links(n, costs, config, output): +def simplify_links(n, costs, config, output, aggregation_strategies=dict()): ## Complex multi-node links are folded into end-points logger.info("Simplifying connected link components") @@ -306,17 +314,19 @@ def simplify_links(n, costs, config, output): logger.debug("Collecting all components using the busmap") - _aggregate_and_move_components(n, busmap, connection_costs_to_bus, output) + _aggregate_and_move_components(n, busmap, connection_costs_to_bus, output, + aggregation_strategies=aggregation_strategies) return n, busmap -def remove_stubs(n, costs, config, output): +def remove_stubs(n, costs, config, output, aggregation_strategies=dict()): logger.info("Removing stubs") busmap = busmap_by_stubs(n) # ['country']) connection_costs_to_bus = _compute_connection_costs_to_bus(n, busmap, costs, config) - _aggregate_and_move_components(n, busmap, connection_costs_to_bus, output) + _aggregate_and_move_components(n, busmap, connection_costs_to_bus, output, + aggregation_strategies=aggregation_strategies) return n, busmap @@ -345,13 +355,14 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None): busmap = n.buses.index.to_series() busmap.loc[buses_i] = dist.idxmin(1) - # default aggregation strategies must be specified within the function, otherwise (when defaults - # are passed in the function's definition) they get lost in case custom values for different - # variables are specified in the config. + # default aggregation strategies that cannot be defined in .yaml format must be specified within + # the function, otherwise (when defaults are passed in the function's definition) they get lost + # in case custom values for different variables are specified in the config. bus_strategies = dict(country=_make_consense("Bus", "country")) bus_strategies.update(aggregation_strategies.get("buses", {})) - generator_strategies = aggregation_strategies.get("generators", {"p_nom_max": "sum"}) + generator_strategies = {'build_year': lambda x: 0, 'lifetime': lambda x: np.inf} + generator_strategies.update(aggregation_strategies.get("generators", {})) clustering = get_clustering_from_busmap(n, busmap, bus_strategies=bus_strategies, @@ -403,9 +414,11 @@ if __name__ == "__main__": technology_costs = load_costs(snakemake.input.tech_costs, snakemake.config['costs'], snakemake.config['electricity'], Nyears) - n, simplify_links_map = simplify_links(n, technology_costs, snakemake.config, snakemake.output) + n, simplify_links_map = simplify_links(n, technology_costs, snakemake.config, snakemake.output, + aggregation_strategies) - n, stub_map = remove_stubs(n, technology_costs, snakemake.config, snakemake.output) + n, stub_map = remove_stubs(n, technology_costs, snakemake.config, snakemake.output, + aggregation_strategies=aggregation_strategies) busmaps = [trafo_map, simplify_links_map, stub_map] diff --git a/test/config.test1.yaml b/test/config.test1.yaml index e3f39ab5..6d626f7e 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -30,6 +30,7 @@ clustering: committable: any ramp_limit_up: max ramp_limit_down: max + efficiency: mean snapshots: start: "2013-03-01" From 2dfa2753ba8b5de6df56d6e6f9b278aa32047d55 Mon Sep 17 00:00:00 2001 From: martacki Date: Mon, 27 Jun 2022 13:59:04 +0200 Subject: [PATCH 6/7] clustering strats to configurables: config spacing --- config.tutorial.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 31ca7f99..edf0091c 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -31,7 +31,7 @@ clustering: committable: any ramp_limit_up: max ramp_limit_down: max - efficiency:mean + efficiency: mean snapshots: start: "2013-03-01" From a3af137b74d6e505e2dae32fc7cea2b2226f834c Mon Sep 17 00:00:00 2001 From: martacki Date: Mon, 27 Jun 2022 14:18:47 +0200 Subject: [PATCH 7/7] clustering strats to configurables: move duplicate code to _helpers script & import --- scripts/_helpers.py | 16 ++++++++++++++++ scripts/cluster_network.py | 8 ++------ scripts/simplify_network.py | 14 +++----------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 6e47c053..3c3e6ac7 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -210,6 +210,22 @@ def progress_retrieve(url, file): urllib.request.urlretrieve(url, file, reporthook=dlProgress) +def get_aggregation_strategies(aggregation_strategies): + # default aggregation strategies that cannot be defined in .yaml format must be specified within + # the function, otherwise (when defaults are passed in the function's definition) they get lost + # when custom values are specified in the config. + + import numpy as np + from pypsa.networkclustering import _make_consense + + bus_strategies = dict(country=_make_consense("Bus", "country")) + bus_strategies.update(aggregation_strategies.get("buses", {})) + + generator_strategies = {'build_year': lambda x: 0, 'lifetime': lambda x: np.inf} + generator_strategies.update(aggregation_strategies.get("generators", {})) + + return bus_strategies, generator_strategies + def mock_snakemake(rulename, **wildcards): """ diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 93cd89cd..6186a00e 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -121,7 +121,7 @@ Exemplary unsolved network clustered to 37 nodes: """ import logging -from _helpers import configure_logging, update_p_nom_max +from _helpers import configure_logging, update_p_nom_max, get_aggregation_strategies import pypsa import os @@ -261,11 +261,7 @@ def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carr line_length_factor=1.25, aggregation_strategies=dict(), solver_name="cbc", algorithm="kmeans", extended_link_costs=0, focus_weights=None): - bus_strategies = dict(country=_make_consense("Bus", "country")) - bus_strategies.update(aggregation_strategies.get("buses", {})) - - generator_strategies = {'build_year': lambda x: 0, 'lifetime': lambda x: np.inf} - generator_strategies.update(aggregation_strategies.get("generators", {})) + bus_strategies, generator_strategies = get_aggregation_strategies(aggregation_strategies) if not isinstance(custom_busmap, pd.Series): busmap = busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights, algorithm) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 0fda5c77..37e7e698 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -83,7 +83,7 @@ The rule :mod:`simplify_network` does up to four things: """ import logging -from _helpers import configure_logging, update_p_nom_max +from _helpers import configure_logging, update_p_nom_max, get_aggregation_strategies from cluster_network import clustering_for_n_clusters, cluster_regions from add_electricity import load_costs @@ -203,8 +203,7 @@ def _aggregate_and_move_components(n, busmap, connection_costs_to_bus, output, _adjust_capital_costs_using_connection_costs(n, connection_costs_to_bus, output) - generator_strategies = {'build_year': lambda x: 0, 'lifetime': lambda x: np.inf} - generator_strategies.update(aggregation_strategies.get("generators", {})) + _, generator_strategies = get_aggregation_strategies(aggregation_strategies) generators, generators_pnl = aggregategenerators( n, busmap, custom_strategies=generator_strategies @@ -355,14 +354,7 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None): busmap = n.buses.index.to_series() busmap.loc[buses_i] = dist.idxmin(1) - # default aggregation strategies that cannot be defined in .yaml format must be specified within - # the function, otherwise (when defaults are passed in the function's definition) they get lost - # in case custom values for different variables are specified in the config. - bus_strategies = dict(country=_make_consense("Bus", "country")) - bus_strategies.update(aggregation_strategies.get("buses", {})) - - generator_strategies = {'build_year': lambda x: 0, 'lifetime': lambda x: np.inf} - generator_strategies.update(aggregation_strategies.get("generators", {})) + bus_strategies, generator_strategies = get_aggregation_strategies(aggregation_strategies) clustering = get_clustering_from_busmap(n, busmap, bus_strategies=bus_strategies,