From ea8e643dc4916e0150e888b0df3152c664f81f98 Mon Sep 17 00:00:00 2001 From: eb5194 Date: Tue, 19 Jan 2021 15:20:58 +0100 Subject: [PATCH 01/10] add_electricity.py Resolve FutureWarning 771 Index.__or__ operating as set operation is deprecated --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 8fc8ad5c..3f151977 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -332,7 +332,7 @@ def attach_hydro(n, costs, ppl): country = ppl['bus'].map(n.buses.country).rename("country") - inflow_idx = ror.index | hydro.index + inflow_idx = ror.index.union(hydro.index) if not inflow_idx.empty: dist_key = ppl.loc[inflow_idx, 'p_nom'].groupby(country).transform(normed) From f3f587e3f8ea7211c6605725fe47ae05fa380216 Mon Sep 17 00:00:00 2001 From: martacki Date: Fri, 21 May 2021 13:54:38 +0200 Subject: [PATCH 02/10] simplify to substations - initial draft --- config.default.yaml | 2 ++ config.tutorial.yaml | 2 ++ scripts/simplify_network.py | 47 +++++++++++++++++++++++++++++++++++-- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index b1111d5a..e210320f 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -59,6 +59,8 @@ electricity: # Wind: [onwind, offwind-ac, offwind-dc] # Solar: [solar] +simplify_to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + atlite: nprocesses: 4 cutouts: diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 1dfde199..97c5d9ca 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -51,6 +51,8 @@ electricity: custom_powerplants: false # use pandas query strings here, e.g. Country in ['Germany'] conventional_carriers: [coal, CCGT] # [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] +simplify_to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + atlite: nprocesses: 4 cutouts: diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index b05d59aa..4572e664 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -98,7 +98,7 @@ from six.moves import reduce import pypsa from pypsa.io import import_components_from_dataframe, import_series_from_dataframe -from pypsa.networkclustering import busmap_by_stubs, aggregategenerators, aggregateoneport +from pypsa.networkclustering import busmap_by_stubs, aggregategenerators, aggregateoneport, get_clustering_from_busmap, _make_consense logger = logging.getLogger(__name__) @@ -308,7 +308,6 @@ def simplify_links(n): _aggregate_and_move_components(n, busmap, connection_costs_to_bus) return n, busmap - def remove_stubs(n): logger.info("Removing stubs") @@ -320,6 +319,46 @@ def remove_stubs(n): return n, busmap +def aggregate_to_substations(n): + logger.info("Aggregating buses that are no substations or have a no valid offshore connection")# + + busmap = n.buses.index.to_series() + + no_substations = list(set(n.buses.index)-set(n.generators.bus)-set(n.loads.bus)) + + + index = [np.append(["Line" for c in range(len(n.lines))], + ["Link" for c in range(len(n.links))]), + np.append(n.lines.index, n.links.index)] + #under_construction lines should be last choice, but weight should be < inf in case no other node is reachable, hence 1e-3 + weight = pd.Series(np.append((n.lines.length/n.lines.s_nom.apply(lambda b: b if b>0 else 1e-3)).values, + (n.links.length/n.links.p_nom.apply(lambda b: b if b>0 else 1e-3)).values), + index=index) + + adj = n.adjacency_matrix(branch_components=['Line', 'Link'], weights=weight) + + dist = dijkstra(adj, directed=False, indices=n.buses.index.get_indexer(no_substations)) + dist[:, n.buses.index.get_indexer(no_substations)] = np.inf #no_substations should not be assigned to other no_substations + + #restrict to same country: + for bus in no_substations: + country_buses = n.buses[~n.buses.country.isin([n.buses.loc[bus].country])].index + dist[n.buses.loc[no_substations].index.get_indexer([bus]),n.buses.index.get_indexer(country_buses)] = np.inf + + assign_to = dist.argmin(axis=1) + busmap.loc[no_substations] = n.buses.iloc[assign_to].index + + clustering = get_clustering_from_busmap(n, busmap, + bus_strategies=dict(country=_make_consense("Bus", "country")), + aggregate_generators_weighted=True, + aggregate_generators_carriers=None, + aggregate_one_ports=["Load", "StorageUnit"], + line_length_factor=1.0, + generator_strategies={'p_nom_max': 'sum'}, + scale_link_capital_costs=False) + + return clustering.network, busmap + def cluster(n, n_clusters): logger.info(f"Clustering to {n_clusters} buses") @@ -358,6 +397,10 @@ if __name__ == "__main__": busmaps = [trafo_map, simplify_links_map, stub_map] + if snakemake.config['simplify_to_substations']: + n, substation_map = aggregate_to_substations(n) + busmaps.append(substation_map) + if snakemake.wildcards.simpl: n, cluster_map = cluster(n, int(snakemake.wildcards.simpl)) busmaps.append(cluster_map) From c53d49d8f82dd027138755e560282e8ffdbcc9ab Mon Sep 17 00:00:00 2001 From: martacki Date: Fri, 28 May 2021 14:53:00 +0200 Subject: [PATCH 03/10] adapt config for test --- test/config.test1.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/config.test1.yaml b/test/config.test1.yaml index d13a6844..75752c4d 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -50,6 +50,8 @@ electricity: custom_powerplants: false # use pandas query strings here, e.g. Country in ['Germany'] conventional_carriers: [coal, CCGT] # [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] +simplify_to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + atlite: nprocesses: 4 cutouts: From d6830d3c482a4eb6c3f23f671e81081d740e3a50 Mon Sep 17 00:00:00 2001 From: martacki Date: Tue, 1 Jun 2021 10:55:26 +0200 Subject: [PATCH 04/10] rename no_substations --- scripts/simplify_network.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 4572e664..72d0dbb7 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -319,14 +319,16 @@ def remove_stubs(n): return n, busmap -def aggregate_to_substations(n): - logger.info("Aggregating buses that are no substations or have a no valid offshore connection")# +def aggregate_to_substations(n, buses_i=None): + # can be used to aggregate a selection of buses to electrically closest neighbors + # if no buses are given, nodes that are no substations or without offshore connection are aggregated + + if buses_i is None: + logger.info("Aggregating buses that are no substations or have no valid offshore connection") + buses_i = list(set(n.buses.index)-set(n.generators.bus)-set(n.loads.bus)) busmap = n.buses.index.to_series() - no_substations = list(set(n.buses.index)-set(n.generators.bus)-set(n.loads.bus)) - - index = [np.append(["Line" for c in range(len(n.lines))], ["Link" for c in range(len(n.links))]), np.append(n.lines.index, n.links.index)] @@ -337,16 +339,16 @@ def aggregate_to_substations(n): adj = n.adjacency_matrix(branch_components=['Line', 'Link'], weights=weight) - dist = dijkstra(adj, directed=False, indices=n.buses.index.get_indexer(no_substations)) - dist[:, n.buses.index.get_indexer(no_substations)] = np.inf #no_substations should not be assigned to other no_substations + dist = dijkstra(adj, directed=False, indices=n.buses.index.get_indexer(buses_i)) + dist[:, n.buses.index.get_indexer(buses_i)] = np.inf #bus in buses_i should not be assigned to different bus in buses_i #restrict to same country: - for bus in no_substations: + for bus in buses_i: country_buses = n.buses[~n.buses.country.isin([n.buses.loc[bus].country])].index - dist[n.buses.loc[no_substations].index.get_indexer([bus]),n.buses.index.get_indexer(country_buses)] = np.inf + dist[n.buses.loc[buses_i].index.get_indexer([bus]),n.buses.index.get_indexer(country_buses)] = np.inf assign_to = dist.argmin(axis=1) - busmap.loc[no_substations] = n.buses.iloc[assign_to].index + busmap.loc[buses_i] = n.buses.iloc[assign_to].index clustering = get_clustering_from_busmap(n, busmap, bus_strategies=dict(country=_make_consense("Bus", "country")), From eef8e3fe41095ceb0ffa7d3815ef81389a35023b Mon Sep 17 00:00:00 2001 From: martacki Date: Tue, 1 Jun 2021 11:00:31 +0200 Subject: [PATCH 05/10] different line in configuration settings --- config.default.yaml | 4 ++-- config.tutorial.yaml | 4 ++-- test/config.test1.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index e210320f..6df2c446 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -19,6 +19,8 @@ scenario: countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'] +simplify_to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + snapshots: start: "2013-01-01" end: "2014-01-01" @@ -59,8 +61,6 @@ electricity: # Wind: [onwind, offwind-ac, offwind-dc] # Solar: [solar] -simplify_to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) - atlite: nprocesses: 4 cutouts: diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 97c5d9ca..4ddb6830 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -19,6 +19,8 @@ scenario: countries: ['DE'] +simplify_to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + snapshots: start: "2013-03-01" end: "2013-04-01" @@ -51,8 +53,6 @@ electricity: custom_powerplants: false # use pandas query strings here, e.g. Country in ['Germany'] conventional_carriers: [coal, CCGT] # [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] -simplify_to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) - atlite: nprocesses: 4 cutouts: diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 75752c4d..39b2ef8e 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -18,6 +18,8 @@ scenario: countries: ['DE'] +simplify_to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) + snapshots: start: "2013-03-01" end: "2014-04-01" @@ -50,8 +52,6 @@ electricity: custom_powerplants: false # use pandas query strings here, e.g. Country in ['Germany'] conventional_carriers: [coal, CCGT] # [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] -simplify_to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) - atlite: nprocesses: 4 cutouts: From 9825fa32b24a1018472555d66cb8a0615278353c Mon Sep 17 00:00:00 2001 From: martacki Date: Tue, 1 Jun 2021 12:00:55 +0200 Subject: [PATCH 06/10] change layout of configuration settings --- config.default.yaml | 4 +++- config.tutorial.yaml | 4 +++- scripts/simplify_network.py | 2 +- test/config.test1.yaml | 4 +++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index 6df2c446..51721729 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -19,7 +19,9 @@ scenario: countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'] -simplify_to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) +clustering: + simplify: + to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) snapshots: start: "2013-01-01" diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 4ddb6830..e551e460 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -19,7 +19,9 @@ scenario: countries: ['DE'] -simplify_to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) +clustering: + simplify: + to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) snapshots: start: "2013-03-01" diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 72d0dbb7..309e94ff 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -399,7 +399,7 @@ if __name__ == "__main__": busmaps = [trafo_map, simplify_links_map, stub_map] - if snakemake.config['simplify_to_substations']: + if snakemake.config['clustering']['simplify']['to_substations']: n, substation_map = aggregate_to_substations(n) busmaps.append(substation_map) diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 39b2ef8e..a5dadc65 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -18,7 +18,9 @@ scenario: countries: ['DE'] -simplify_to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) +clustering: + simplify: + to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections) snapshots: start: "2013-03-01" From cec9dcc41ce96034c6c11f78d154b06b04be759f Mon Sep 17 00:00:00 2001 From: martacki Date: Tue, 1 Jun 2021 12:02:43 +0200 Subject: [PATCH 07/10] adapt configuration options --- doc/configtables/clustering.csv | 3 +++ doc/configuration.rst | 6 +++--- doc/tutorial.rst | 12 ++++++------ 3 files changed, 12 insertions(+), 9 deletions(-) create mode 100644 doc/configtables/clustering.csv diff --git a/doc/configtables/clustering.csv b/doc/configtables/clustering.csv new file mode 100644 index 00000000..2f63f955 --- /dev/null +++ b/doc/configtables/clustering.csv @@ -0,0 +1,3 @@ +,Unit,Values,Description +simplify,,, +-- to_substations,bool,"{'true','false'}","Aggregates all nodes without power injection (positive or negative, i.e. demand or generation) to electrically closest ones" diff --git a/doc/configuration.rst b/doc/configuration.rst index a75669cd..e7c31793 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -256,7 +256,7 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 192-202 + :lines: 208-217 .. csv-table:: :header-rows: 1 @@ -268,7 +268,7 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 192,203-219 + :lines: 218-227 .. csv-table:: :header-rows: 1 @@ -282,7 +282,7 @@ Define and specify the ``atlite.Cutout`` used for calculating renewable potentia .. literalinclude:: ../config.default.yaml :language: yaml - :lines: 221-299 + :lines: 236-314 .. csv-table:: :header-rows: 1 diff --git a/doc/tutorial.rst b/doc/tutorial.rst index 507b1485..d2fb8433 100644 --- a/doc/tutorial.rst +++ b/doc/tutorial.rst @@ -53,41 +53,41 @@ Likewise, the example's temporal scope can be restricted (e.g. to a single month .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 22-25 + :lines: 24-27 It is also possible to allow less or more carbon-dioxide emissions. Here, we limit the emissions of Germany 100 Megatonnes per year. .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 36,38 + :lines: 38,40 PyPSA-Eur also includes a database of existing conventional powerplants. We can select which types of powerplants we like to be included with fixed capacities: .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 36,52 + :lines: 38,54 To accurately model the temporal and spatial availability of renewables such as wind and solar energy, we rely on historical weather data. It is advisable to adapt the required range of coordinates to the selection of countries. .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 54-62 + :lines: 56-63 We can also decide which weather data source should be used to calculate potentials and capacity factor time-series for each carrier. For example, we may want to use the ERA-5 dataset for solar and not the default SARAH-2 dataset. .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 64,107-108 + :lines: 65,108-109 Finally, it is possible to pick a solver. For instance, this tutorial uses the open-source solvers CBC and Ipopt and does not rely on the commercial solvers Gurobi or CPLEX (for which free academic licenses are available). .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 170,180-181 + :lines: 171,181-182 .. note:: From 0f5934f534c857fbbfb9b90f87ca6b8606f061fd Mon Sep 17 00:00:00 2001 From: martacki Date: Tue, 1 Jun 2021 12:06:56 +0200 Subject: [PATCH 08/10] release notes --- doc/release_notes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index a1b54396..5330128c 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -19,6 +19,7 @@ Upcoming Release * Fix: Value for ``co2base`` in ``config.yaml`` adjusted to 1.487e9 t CO2-eq (from 3.1e9 t CO2-eq). The new value represents emissions related to the electricity sector for EU+UK. The old value was ~2x too high and used when the emissions wildcard in ``{opts}`` was used. * Add option to include marginal costs of links representing fuel cells, electrolysis, and battery inverters [`#232 `_]. +* Add option to pre-aggregate nodes without power injections (positive or negative, i.e. generation or demand) to electrically closest nodes or neighbors in ``simplify_network``. Defaults to ``False``. This affects nodes that are no substations or have no offshore connection. PyPSA-Eur 0.3.0 (7th December 2020) ================================== From a6d2a0a99d42d6647565e05675a99fae081e289b Mon Sep 17 00:00:00 2001 From: Martha Frysztacki Date: Thu, 26 Aug 2021 16:00:08 +0200 Subject: [PATCH 09/10] style Co-authored-by: Fabian Hofmann --- scripts/simplify_network.py | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 82e89eff..17c23a43 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -331,28 +331,22 @@ def aggregate_to_substations(n, buses_i=None): logger.info("Aggregating buses that are no substations or have no valid offshore connection") buses_i = list(set(n.buses.index)-set(n.generators.bus)-set(n.loads.bus)) - busmap = n.buses.index.to_series() - - index = [np.append(["Line" for c in range(len(n.lines))], - ["Link" for c in range(len(n.links))]), - np.append(n.lines.index, n.links.index)] - #under_construction lines should be last choice, but weight should be < inf in case no other node is reachable, hence 1e-3 - weight = pd.Series(np.append((n.lines.length/n.lines.s_nom.apply(lambda b: b if b>0 else 1e-3)).values, - (n.links.length/n.links.p_nom.apply(lambda b: b if b>0 else 1e-3)).values), - index=index) + weight = pd.concat({'Line': n.lines.length/n.lines.s_nom.clip(1e-3), + 'Link': n.links.length/n.links.p_nom.clip(1e-3)}) adj = n.adjacency_matrix(branch_components=['Line', 'Link'], weights=weight) - dist = dijkstra(adj, directed=False, indices=n.buses.index.get_indexer(buses_i)) - dist[:, n.buses.index.get_indexer(buses_i)] = np.inf #bus in buses_i should not be assigned to different bus in buses_i + bus_indexer = n.buses.index.get_indexer(buses_i) + dist = pd.DataFrame(dijkstra(adj, directed=False, indices=bus_indexer), buses_i, n.buses.index) - #restrict to same country: - for bus in buses_i: - country_buses = n.buses[~n.buses.country.isin([n.buses.loc[bus].country])].index - dist[n.buses.loc[buses_i].index.get_indexer([bus]),n.buses.index.get_indexer(country_buses)] = np.inf - - assign_to = dist.argmin(axis=1) - busmap.loc[buses_i] = n.buses.iloc[assign_to].index + dist[buses_i] = np.inf # bus in buses_i should not be assigned to different bus in buses_i + + for c in n.buses.country.unique(): + incountry_b = n.buses.country == c + dist.loc[incountry_b, ~incountry_b] = np.inf + + busmap = n.buses.index.to_series() + busmap.loc[buses_i] = dist.idxmin(1) clustering = get_clustering_from_busmap(n, busmap, bus_strategies=dict(country=_make_consense("Bus", "country")), From b9bbefa09f32f35f41c75eb85e29a592db4a1f55 Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Fri, 27 Aug 2021 10:34:27 +0200 Subject: [PATCH 10/10] simplify_network.py fix dict getting --- scripts/simplify_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 17c23a43..6e12e5e8 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -400,7 +400,7 @@ if __name__ == "__main__": busmaps = [trafo_map, simplify_links_map, stub_map] - if snakemake.config['clustering']['simplify']['to_substations']: + if snakemake.config.get('clustering', {}).get('simplify', {}).get('to_substations', False): n, substation_map = aggregate_to_substations(n) busmaps.append(substation_map)