From 8700f63cdb7c6be02faa8eeba66718ece67c3c3e Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 3 Mar 2022 18:13:54 +0100 Subject: [PATCH 01/54] solve_network: fix load shedding attributes --- scripts/solve_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index b902f525..c8c44969 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -100,7 +100,7 @@ def prepare_network(n, solve_opts): df.where(df>solve_opts['clip_p_max_pu'], other=0., inplace=True) if solve_opts.get('load_shedding'): - n.add("Carrier", "Load") + n.add("Carrier", "load", color='red', nice_name="Load shedding") buses_i = n.buses.query("carrier == 'AC'").index n.madd("Generator", buses_i, " load", bus=buses_i, From d16f24fc07db277b68eb2f354c26a9748686ab1d Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 3 Mar 2022 23:08:29 +0100 Subject: [PATCH 02/54] follow up --- scripts/solve_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index c8c44969..46df0328 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -100,7 +100,7 @@ def prepare_network(n, solve_opts): df.where(df>solve_opts['clip_p_max_pu'], other=0., inplace=True) if solve_opts.get('load_shedding'): - n.add("Carrier", "load", color='red', nice_name="Load shedding") + n.add("Carrier", "load", color="#dd2e23", nice_name="Load shedding") buses_i = n.buses.query("carrier == 'AC'").index n.madd("Generator", buses_i, " load", bus=buses_i, From 0384b4ff834f4d2141c9c1e5c29dd1c649258a6a Mon Sep 17 00:00:00 2001 From: Seth Date: Thu, 10 Mar 2022 13:55:53 +0100 Subject: [PATCH 03/54] fix the plot_network snakemake rule --- scripts/_helpers.py | 2 +- scripts/plot_network.py | 25 +++++++++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index f1e5e887..410e05af 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -120,7 +120,7 @@ def load_network_for_plots(fn, tech_costs, config, combine_hydro_ps=True): # n.storage_units.loc[bus_carrier == "heat","carrier"] = "water tanks" Nyears = n.snapshot_weightings.objective.sum() / 8760. - costs = load_costs(Nyears, tech_costs, config['costs'], config['electricity']) + costs = load_costs(tech_costs, config['costs'], config['electricity'], Nyears) update_transmission_costs(n, costs) return n diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 645c8c39..24d49473 100755 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -20,8 +20,7 @@ Description """ import logging -from _helpers import (retrieve_snakemake_keys, load_network_for_plots, - aggregate_p, aggregate_costs, configure_logging) +from _helpers import (load_network_for_plots, aggregate_p, aggregate_costs, configure_logging) import pandas as pd import numpy as np @@ -182,7 +181,7 @@ def plot_map(n, ax=None, attribute='p_nom', opts={}): return fig -def plot_total_energy_pie(n, ax=None): +def plot_total_energy_pie(n, ax=None, opts={}): if ax is None: ax = plt.gca() ax.set_title('Energy per technology', fontdict=dict(fontsize="medium")) @@ -200,7 +199,7 @@ def plot_total_energy_pie(n, ax=None): t1.remove() t2.remove() -def plot_total_cost_bar(n, ax=None): +def plot_total_cost_bar(n, ax=None, opts={}): if ax is None: ax = plt.gca() total_load = (n.snapshot_weightings.generators * n.loads_t.p.sum(axis=1)).sum() @@ -259,25 +258,31 @@ if __name__ == "__main__": set_plot_style() - paths, config, wildcards, logs, out = retrieve_snakemake_keys(snakemake) + paths, config, wildcards, logs, out = ( + snakemake.input, + snakemake.config, + snakemake.wildcards, + snakemake.log, + snakemake.output, + ) - map_figsize = config['map']['figsize'] - map_boundaries = config['map']['boundaries'] + map_figsize = config["plotting"]['map']['figsize'] + map_boundaries = config["plotting"]['map']['boundaries'] n = load_network_for_plots(paths.network, paths.tech_costs, config) scenario_opts = wildcards.opts.split('-') fig, ax = plt.subplots(figsize=map_figsize, subplot_kw={"projection": ccrs.PlateCarree()}) - plot_map(n, ax, wildcards.attr, config) + plot_map(n, ax, wildcards.attr, config["plotting"]) fig.savefig(out.only_map, dpi=150, bbox_inches='tight') ax1 = fig.add_axes([-0.115, 0.625, 0.2, 0.2]) - plot_total_energy_pie(n, ax1) + plot_total_energy_pie(n, ax1, config["plotting"]) ax2 = fig.add_axes([-0.075, 0.1, 0.1, 0.45]) - plot_total_cost_bar(n, ax2) + plot_total_cost_bar(n, ax2, config["plotting"]) ll = wildcards.ll ll_type = ll[0] From 65e35135e18a2eab21fe4c8e4de17fe0e6278235 Mon Sep 17 00:00:00 2001 From: euronion <42553970+euronion@users.noreply.github.com> Date: Fri, 18 Mar 2022 11:13:58 +0100 Subject: [PATCH 04/54] Update Snakefile --- Snakefile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Snakefile b/Snakefile index 4b8fa0b3..a2d4c4b2 100644 --- a/Snakefile +++ b/Snakefile @@ -201,19 +201,19 @@ rule build_renewable_profiles: benchmark: "benchmarks/build_renewable_profiles_{technology}" threads: ATLITE_NPROCESSES resources: mem_mb=ATLITE_NPROCESSES * 5000 + wildcard_constraints: technology="solar|onwind|offwind-ac|offwind-dc" script: "scripts/build_renewable_profiles.py" -if 'hydro' in config['renewable'].keys(): - rule build_hydro_profile: - input: - country_shapes='resources/country_shapes.geojson', - eia_hydro_generation='data/bundle/EIA_hydro_generation_2000_2014.csv', - cutout="cutouts/" + config["renewable"]['hydro']['cutout'] + ".nc" - output: 'resources/profile_hydro.nc' - log: "logs/build_hydro_profile.log" - resources: mem_mb=5000 - script: 'scripts/build_hydro_profile.py' +rule build_hydro_profile: + input: + country_shapes='resources/country_shapes.geojson', + eia_hydro_generation='data/bundle/EIA_hydro_generation_2000_2014.csv', + cutout="cutouts/" + config["renewable"]['hydro']['cutout'] + ".nc" + output: 'resources/profile_hydro.nc' + log: "logs/build_hydro_profile.log" + resources: mem_mb=5000 + script: 'scripts/build_hydro_profile.py' rule add_electricity: From da72f5487dbfd247b49a4c652378e41619dd1214 Mon Sep 17 00:00:00 2001 From: euronion <42553970+euronion@users.noreply.github.com> Date: Fri, 18 Mar 2022 14:29:50 +0100 Subject: [PATCH 05/54] Update Snakefile --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index a2d4c4b2..5286aa06 100644 --- a/Snakefile +++ b/Snakefile @@ -209,7 +209,7 @@ rule build_hydro_profile: input: country_shapes='resources/country_shapes.geojson', eia_hydro_generation='data/bundle/EIA_hydro_generation_2000_2014.csv', - cutout="cutouts/" + config["renewable"]['hydro']['cutout'] + ".nc" + cutout=f"cutouts/{config['renewable']['hydro']['cutout']}.nc" if "hydro" in config["renewable"] else "config['renewable']['hydro']['cutout'] not configured", output: 'resources/profile_hydro.nc' log: "logs/build_hydro_profile.log" resources: mem_mb=5000 From ca94709ed3621e7f75bdfcb77b1133e8ffb83306 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sun, 20 Mar 2022 09:50:38 +0100 Subject: [PATCH 06/54] use snakemake.threads in build_renewable_profiles --- scripts/build_renewable_profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 36845da5..a2b2eda6 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -202,7 +202,7 @@ if __name__ == '__main__': configure_logging(snakemake) pgb.streams.wrap_stderr() - nprocesses = snakemake.config['atlite'].get('nprocesses') + nprocesses = int(snakemake.threads) noprogress = not snakemake.config['atlite'].get('show_progress', True) config = snakemake.config['renewable'][snakemake.wildcards.technology] resource = config['resource'] # pv panel config / wind turbine config From 30cb861ca2644e4c1acc3e1a7425c9e746bcb46b Mon Sep 17 00:00:00 2001 From: euronion <42553970+euronion@users.noreply.github.com> Date: Mon, 21 Mar 2022 09:37:50 +0100 Subject: [PATCH 07/54] Update Snakefile --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index 5286aa06..dc8277ea 100644 --- a/Snakefile +++ b/Snakefile @@ -201,7 +201,7 @@ rule build_renewable_profiles: benchmark: "benchmarks/build_renewable_profiles_{technology}" threads: ATLITE_NPROCESSES resources: mem_mb=ATLITE_NPROCESSES * 5000 - wildcard_constraints: technology="solar|onwind|offwind-ac|offwind-dc" + wildcard_constraints: technology="^(?!hydro).*$" # Any technology other than hydro script: "scripts/build_renewable_profiles.py" From e2e98120b17b3ae2c6137d7257dc17f6a05aa0b9 Mon Sep 17 00:00:00 2001 From: euronion <42553970+euronion@users.noreply.github.com> Date: Mon, 21 Mar 2022 10:01:34 +0100 Subject: [PATCH 08/54] Update Snakefile --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index dc8277ea..a0709043 100644 --- a/Snakefile +++ b/Snakefile @@ -201,7 +201,7 @@ rule build_renewable_profiles: benchmark: "benchmarks/build_renewable_profiles_{technology}" threads: ATLITE_NPROCESSES resources: mem_mb=ATLITE_NPROCESSES * 5000 - wildcard_constraints: technology="^(?!hydro).*$" # Any technology other than hydro + wildcard_constraints: technology="(?!hydro).*" # Any technology other than hydro script: "scripts/build_renewable_profiles.py" From 1a82b875872cb082c751fb6a436b547e48f5ba54 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 24 Mar 2022 14:30:28 +0100 Subject: [PATCH 09/54] skip iterations if no lines are expandable --- scripts/solve_network.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index b902f525..4704d179 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -254,7 +254,12 @@ def solve_network(n, config, opts='', **kwargs): n.config = config n.opts = opts - if cf_solving.get('skip_iterations', False): + skip_iterations = cf_solving.get('skip_iterations', False) + if not n.lines.s_nom_extendable.any(): + skip_iterations = True + logger.info("No expandable lines found. Skipping iterative solving.") + + if skip_iterations: network_lopf(n, solver_name=solver_name, solver_options=solver_options, extra_functionality=extra_functionality, **kwargs) else: From 12a5a5f86a1c4e59929234d6cacad1a73018de2c Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 24 Mar 2022 14:34:08 +0100 Subject: [PATCH 10/54] add release notes --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 3b20fbcf..79689bf5 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -62,6 +62,8 @@ Upcoming Release * New network topology extracted from the ENTSO-E interactive map. +* Iterative solving with impedance updates is skipped if there are no expandable lines. + PyPSA-Eur 0.4.0 (22th September 2021) ===================================== From c54ddab4a331fdbe08f4938cb838bf2f128d93f0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 24 Mar 2022 14:47:00 +0100 Subject: [PATCH 11/54] remove unused simple_hvdc_costs --- doc/release_notes.rst | 1 + scripts/add_electricity.py | 18 +++++++----------- scripts/make_summary.py | 2 +- scripts/prepare_network.py | 2 +- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 3b20fbcf..7deee108 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -62,6 +62,7 @@ Upcoming Release * New network topology extracted from the ENTSO-E interactive map. +* The unused argument ``simple_hvdc_costs`` in :mod:`add_electricity` was removed. PyPSA-Eur 0.4.0 (22th September 2021) ===================================== diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 7dffe60f..ad932cd8 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -227,7 +227,7 @@ def attach_load(n, regions, load, nuts3_shapes, countries, scaling=1.): n.madd("Load", substation_lv_i, bus=substation_lv_i, p_set=load) -def update_transmission_costs(n, costs, length_factor=1.0, simple_hvdc_costs=False): +def update_transmission_costs(n, costs, length_factor=1.0): # TODO: line length factor of lines is applied to lines and links. # Separate the function to distinguish. @@ -242,16 +242,12 @@ def update_transmission_costs(n, costs, length_factor=1.0, simple_hvdc_costs=Fal # may be missing. Therefore we have to return here. if n.links.loc[dc_b].empty: return - if simple_hvdc_costs: - costs = (n.links.loc[dc_b, 'length'] * length_factor * - costs.at['HVDC overhead', 'capital_cost']) - else: - costs = (n.links.loc[dc_b, 'length'] * length_factor * - ((1. - n.links.loc[dc_b, 'underwater_fraction']) * - costs.at['HVDC overhead', 'capital_cost'] + - n.links.loc[dc_b, 'underwater_fraction'] * - costs.at['HVDC submarine', 'capital_cost']) + - costs.at['HVDC inverter pair', 'capital_cost']) + costs = (n.links.loc[dc_b, 'length'] * length_factor * + ((1. - n.links.loc[dc_b, 'underwater_fraction']) * + costs.at['HVDC overhead', 'capital_cost'] + + n.links.loc[dc_b, 'underwater_fraction'] * + costs.at['HVDC submarine', 'capital_cost']) + + costs.at['HVDC inverter pair', 'capital_cost']) n.links.loc[dc_b, 'capital_cost'] = costs diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 854e9463..af1ecf36 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -404,7 +404,7 @@ def make_summaries(networks_dict, paths, config, country='all'): Nyears = n.snapshot_weightings.objective.sum() / 8760. costs = load_costs(paths[0], config['costs'], config['electricity'], Nyears) - update_transmission_costs(n, costs, simple_hvdc_costs=False) + update_transmission_costs(n, costs) assign_carriers(n) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index f984ace6..206e220b 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -104,7 +104,7 @@ def set_transmission_limit(n, ll_type, factor, costs, Nyears=1): ref = (lines_s_nom @ n.lines[col] + n.links.loc[links_dc_b, "p_nom"] @ n.links.loc[links_dc_b, col]) - update_transmission_costs(n, costs, simple_hvdc_costs=False) + update_transmission_costs(n, costs) if factor == 'opt' or float(factor) > 1.0: n.lines['s_nom_min'] = lines_s_nom From 35f8b54a460a2014954e7a8e1d7e8a2de89779c2 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 24 Mar 2022 15:51:36 +0100 Subject: [PATCH 12/54] switch to Belgium for CI and tutorial --- Snakefile | 2 +- config.tutorial.yaml | 10 +++++----- test/config.test1.yaml | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Snakefile b/Snakefile index a0709043..7678a401 100644 --- a/Snakefile +++ b/Snakefile @@ -160,7 +160,7 @@ if config['enable'].get('build_cutout', False): if config['enable'].get('retrieve_cutout', True): rule retrieve_cutout: - input: HTTP.remote("zenodo.org/record/4709858/files/{cutout}.nc", keep_local=True, static=True) + input: HTTP.remote("zenodo.org/record/6382570/files/{cutout}.nc", keep_local=True, static=True) output: "cutouts/{cutout}.nc" run: move(input[0], output[0]) diff --git a/config.tutorial.yaml b/config.tutorial.yaml index ea624727..5cade102 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -58,7 +58,7 @@ electricity: atlite: nprocesses: 4 cutouts: - europe-2013-era5-tutorial: + be-03-2013-era5: module: era5 x: [4., 15.] y: [46., 56.] @@ -66,7 +66,7 @@ atlite: renewable: onwind: - cutout: europe-2013-era5-tutorial + cutout: be-03-2013-era5 resource: method: wind turbine: Vestas_V112_3MW @@ -83,7 +83,7 @@ renewable: potential: simple # or conservative clip_p_max_pu: 1.e-2 offwind-ac: - cutout: europe-2013-era5-tutorial + cutout: be-03-2013-era5 resource: method: wind turbine: NREL_ReferenceTurbine_5MW_offshore @@ -95,7 +95,7 @@ renewable: potential: simple # or conservative clip_p_max_pu: 1.e-2 offwind-dc: - cutout: europe-2013-era5-tutorial + cutout: be-03-2013-era5 resource: method: wind turbine: NREL_ReferenceTurbine_5MW_offshore @@ -108,7 +108,7 @@ renewable: potential: simple # or conservative clip_p_max_pu: 1.e-2 solar: - cutout: europe-2013-era5-tutorial + cutout: be-03-2013-era5 resource: method: pv panel: CSi diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 2986037b..27cf739c 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -57,7 +57,7 @@ electricity: atlite: nprocesses: 4 cutouts: - europe-2013-era5-tutorial: + be-03-2013-era5: module: era5 x: [4., 15.] y: [46., 56.] @@ -65,7 +65,7 @@ atlite: renewable: onwind: - cutout: europe-2013-era5-tutorial + cutout: be-03-2013-era5 resource: method: wind turbine: Vestas_V112_3MW @@ -82,7 +82,7 @@ renewable: potential: simple # or conservative clip_p_max_pu: 1.e-2 offwind-ac: - cutout: europe-2013-era5-tutorial + cutout: be-03-2013-era5 resource: method: wind turbine: NREL_ReferenceTurbine_5MW_offshore @@ -94,7 +94,7 @@ renewable: potential: simple # or conservative clip_p_max_pu: 1.e-2 offwind-dc: - cutout: europe-2013-era5-tutorial + cutout: be-03-2013-era5 resource: method: wind turbine: NREL_ReferenceTurbine_5MW_offshore @@ -107,7 +107,7 @@ renewable: potential: simple # or conservative clip_p_max_pu: 1.e-2 solar: - cutout: europe-2013-era5-tutorial + cutout: be-03-2013-era5 resource: method: pv panel: CSi From bf8cbf507eb45f9fae7fded03033f27746ee015b Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 24 Mar 2022 15:52:01 +0100 Subject: [PATCH 13/54] switch to Belgium for CI and tutorial --- test/config.test1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 27cf739c..a9ce1e50 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -16,7 +16,7 @@ scenario: clusters: [5] opts: [Co2L-24H] -countries: ['DE'] +countries: ['BE'] clustering: simplify: From d94719a33db6b1a6007f2c8104729e8bc9f52c42 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 24 Mar 2022 15:53:02 +0100 Subject: [PATCH 14/54] fix tutorial and add release notes --- config.tutorial.yaml | 2 +- doc/release_notes.rst | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 5cade102..225d8f78 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -17,7 +17,7 @@ scenario: clusters: [5] opts: [Co2L-24H] -countries: ['DE'] +countries: ['BE'] clustering: simplify: diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 3b20fbcf..f9f9967b 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -62,6 +62,9 @@ Upcoming Release * New network topology extracted from the ENTSO-E interactive map. +* Switch from Germany to Belgium for continuous integration and tutorial to save resources. + +* Use updated SARAH-2 and ERA5 cutouts with slightly wider scope to east and additional variables. PyPSA-Eur 0.4.0 (22th September 2021) ===================================== @@ -104,7 +107,7 @@ PyPSA-Eur 0.4.0 (22th September 2021) [`#261 `_]. * The tutorial cutout was renamed from ``cutouts/europe-2013-era5.nc`` to - ``cutouts/europe-2013-era5-tutorial.nc`` to accomodate tutorial and productive + ``cutouts/be-03-2013-era5.nc`` to accomodate tutorial and productive cutouts side-by-side. * The flag ``keep_all_available_areas`` in the configuration for renewable From e45c7a65ffb767a5831ac6623813d2dd11228afb Mon Sep 17 00:00:00 2001 From: Seth <78690362+thesethtruth@users.noreply.github.com> Date: Mon, 28 Mar 2022 12:10:51 +0200 Subject: [PATCH 15/54] Apply suggestions from Martha's code review Co-authored-by: Martha Frysztacki --- scripts/plot_network.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 24d49473..3f95e16d 100755 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -181,7 +181,7 @@ def plot_map(n, ax=None, attribute='p_nom', opts={}): return fig -def plot_total_energy_pie(n, ax=None, opts={}): +def plot_total_energy_pie(n, opts, ax=None): if ax is None: ax = plt.gca() ax.set_title('Energy per technology', fontdict=dict(fontsize="medium")) @@ -199,7 +199,7 @@ def plot_total_energy_pie(n, ax=None, opts={}): t1.remove() t2.remove() -def plot_total_cost_bar(n, ax=None, opts={}): +def plot_total_cost_bar(n, opts, ax=None): if ax is None: ax = plt.gca() total_load = (n.snapshot_weightings.generators * n.loads_t.p.sum(axis=1)).sum() @@ -258,13 +258,7 @@ if __name__ == "__main__": set_plot_style() - paths, config, wildcards, logs, out = ( - snakemake.input, - snakemake.config, - snakemake.wildcards, - snakemake.log, - snakemake.output, - ) + config, wildcards = snakemake.config, snakemake.wildcards map_figsize = config["plotting"]['map']['figsize'] map_boundaries = config["plotting"]['map']['boundaries'] From c37171b01d772b0b8cdec19b386cbd4493461faf Mon Sep 17 00:00:00 2001 From: Seth Date: Mon, 28 Mar 2022 13:42:55 +0200 Subject: [PATCH 16/54] feedback code review (opts argument and unpacking) --- scripts/plot_network.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 3f95e16d..71a6e627 100755 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -74,7 +74,7 @@ def set_plot_style(): }]) -def plot_map(n, ax=None, attribute='p_nom', opts={}): +def plot_map(n, opts, ax=None, attribute='p_nom'): if ax is None: ax = plt.gca() @@ -263,20 +263,20 @@ if __name__ == "__main__": map_figsize = config["plotting"]['map']['figsize'] map_boundaries = config["plotting"]['map']['boundaries'] - n = load_network_for_plots(paths.network, paths.tech_costs, config) + n = load_network_for_plots(snakemake.input.network, snakemake.input.tech_costs, config) scenario_opts = wildcards.opts.split('-') fig, ax = plt.subplots(figsize=map_figsize, subplot_kw={"projection": ccrs.PlateCarree()}) - plot_map(n, ax, wildcards.attr, config["plotting"]) + plot_map(n, config["plotting"], ax=ax, attribute=wildcards.attr) - fig.savefig(out.only_map, dpi=150, bbox_inches='tight') + fig.savefig(snakemake.output.only_map, dpi=150, bbox_inches='tight') ax1 = fig.add_axes([-0.115, 0.625, 0.2, 0.2]) - plot_total_energy_pie(n, ax1, config["plotting"]) + plot_total_energy_pie(n, config["plotting"], ax=ax1) ax2 = fig.add_axes([-0.075, 0.1, 0.1, 0.45]) - plot_total_cost_bar(n, ax2, config["plotting"]) + plot_total_cost_bar(n, config["plotting"], ax=ax2) ll = wildcards.ll ll_type = ll[0] @@ -286,4 +286,4 @@ if __name__ == "__main__": fig.suptitle('Expansion to {amount} {label} at {clusters} clusters' .format(amount=amnt, label=lbl, clusters=wildcards.clusters)) - fig.savefig(out.ext, transparent=True, bbox_inches='tight') + fig.savefig(snakemake.output.ext, transparent=True, bbox_inches='tight') From a05881479ccd363cf729e74e9947dcc45fbede33 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 28 Mar 2022 15:17:55 +0200 Subject: [PATCH 17/54] build_bus_regions: move voronoi partition from vresutils to script --- scripts/build_bus_regions.py | 65 ++++++++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index d91d0575..4f2369b6 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -47,9 +47,10 @@ from _helpers import configure_logging import pypsa import os import pandas as pd +import numpy as np import geopandas as gpd - -from vresutils.graph import voronoi_partition_pts +from shapely.geometry import Polygon +from scipy.spatial import Voronoi logger = logging.getLogger(__name__) @@ -61,6 +62,66 @@ def save_to_geojson(s, fn): s.to_file(fn, driver='GeoJSON', schema=schema) +def voronoi_partition_pts(points, outline, no_multipolygons=False): + """ + Compute the polygons of a voronoi partition of `points` within the + polygon `outline`. Taken from + https://github.com/FRESNA/vresutils/blob/master/vresutils/graph.py + Attributes + ---------- + points : Nx2 - ndarray[dtype=float] + outline : Polygon + no_multipolygons : bool (default: False) + If true, replace each MultiPolygon by its largest component + Returns + ------- + polygons : N - ndarray[dtype=Polygon|MultiPolygon] + """ + + points = np.asarray(points) + + if len(points) == 1: + polygons = [outline] + else: + xmin, ymin = np.amin(points, axis=0) + xmax, ymax = np.amax(points, axis=0) + xspan = xmax - xmin + yspan = ymax - ymin + + # to avoid any network positions outside all Voronoi cells, append + # the corners of a rectangle framing these points + vor = Voronoi(np.vstack((points, + [[xmin-3.*xspan, ymin-3.*yspan], + [xmin-3.*xspan, ymax+3.*yspan], + [xmax+3.*xspan, ymin-3.*yspan], + [xmax+3.*xspan, ymax+3.*yspan]]))) + + polygons = [] + for i in range(len(points)): + poly = Polygon(vor.vertices[vor.regions[vor.point_region[i]]]) + + if not poly.is_valid: + poly = poly.buffer(0) + + poly = poly.intersection(outline) + + polygons.append(poly) + + if no_multipolygons: + def demultipolygon(poly): + try: + # for a MultiPolygon pick the part with the largest area + poly = max(poly.geoms, key=lambda pg: pg.area) + except: + pass + return poly + polygons = [demultipolygon(poly) for poly in polygons] + + polygons_arr = np.empty((len(polygons),), 'object') + polygons_arr[:] = polygons + return polygons_arr + + if __name__ == "__main__": if 'snakemake' not in globals(): from _helpers import mock_snakemake From 9eac6d9bbea17f0c8dacf003132df5a942002ff8 Mon Sep 17 00:00:00 2001 From: davide-f Date: Mon, 28 Mar 2022 21:45:03 +0200 Subject: [PATCH 18/54] Fix environment --- envs/environment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/envs/environment.yaml b/envs/environment.yaml index 0c881720..00b7830a 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -37,6 +37,7 @@ dependencies: - pyomo - matplotlib - proj + - fiona <= 1.18.20 # Till issue https://github.com/Toblerity/Fiona/issues/1085 is not solved # Keep in conda environment when calling ipython - ipython From 70640cc336c9ed559c6ea04ed7517c7046b1484a Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sun, 3 Apr 2022 09:42:07 +0200 Subject: [PATCH 19/54] doc: fix tutorial config line references --- doc/tutorial.rst | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/doc/tutorial.rst b/doc/tutorial.rst index 17d4e3c1..76970aff 100644 --- a/doc/tutorial.rst +++ b/doc/tutorial.rst @@ -53,41 +53,43 @@ Likewise, the example's temporal scope can be restricted (e.g. to a single month .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 24-27 + :start-at: snapshots: + :end-before: enable: It is also possible to allow less or more carbon-dioxide emissions. Here, we limit the emissions of Germany 100 Megatonnes per year. .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 38,40 + :lines: 40,42 PyPSA-Eur also includes a database of existing conventional powerplants. We can select which types of powerplants we like to be included with fixed capacities: .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 38,54 + :lines: 40,56 To accurately model the temporal and spatial availability of renewables such as wind and solar energy, we rely on historical weather data. It is advisable to adapt the required range of coordinates to the selection of countries. .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 56-63 + :start-at: atlite: + :end-before: renewable: We can also decide which weather data source should be used to calculate potentials and capacity factor time-series for each carrier. For example, we may want to use the ERA-5 dataset for solar and not the default SARAH-2 dataset. .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 65,108-109 + :lines: 67,110,111 Finally, it is possible to pick a solver. For instance, this tutorial uses the open-source solvers CBC and Ipopt and does not rely on the commercial solvers Gurobi or CPLEX (for which free academic licenses are available). .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 171,181-182 + :lines: 173,183,184 .. note:: @@ -126,11 +128,6 @@ orders ``snakemake`` to run the script ``solve_network`` that produces the solve .. until https://github.com/snakemake/snakemake/issues/46 closed -.. warning:: - On Windows the previous command may currently cause a ``MissingRuleException`` due to problems with output files in subfolders. - This is an `open issue `_ at `snakemake `_. - Windows users should add the option ``--keep-target-files`` to the command or instead run ``snakemake -j 1 solve_all_networks``. - This triggers a workflow of multiple preceding jobs that depend on each rule's inputs and outputs: .. graphviz:: @@ -271,7 +268,8 @@ the wildcards given in ``scenario`` in the configuration file ``config.yaml`` ar .. literalinclude:: ../config.tutorial.yaml :language: yaml - :lines: 14-18 + :start-at: scenario: + :end-before: countries: In this example we would not only solve a 6-node model of Germany but also a 2-node model. From a217415c396b7e1eea58c346b21f865366e15849 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sun, 3 Apr 2022 09:42:35 +0200 Subject: [PATCH 20/54] env: bump minimum version number of atlite due to new cutouts with azimuth wind --- envs/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 00b7830a..3c69b77b 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -11,7 +11,7 @@ dependencies: - pip - pypsa>=0.18.1 - - atlite>=0.2.5 + - atlite>=0.2.6 - dask # Dependencies of the workflow itself From 46acddd3697c9af84e6be5fcfef229e2a32609d3 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 4 Apr 2022 22:17:40 +0200 Subject: [PATCH 21/54] update TYNDP links which are already built --- data/links_tyndp.csv | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/data/links_tyndp.csv b/data/links_tyndp.csv index 8079be72..38203979 100644 --- a/data/links_tyndp.csv +++ b/data/links_tyndp.csv @@ -1,14 +1,15 @@ Name,Converterstation 1,Converterstation 2,Length (given) (km),Length (distance*1.2) (km),Power (MW),status,replaces,Ref,x1,y1,x2,y2 Biscay Gulf,Gatica (ES),Cubnezais (FR),370,,2200,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/16,-2.867,43.367,-0.408943,45.074191 Italy-France,Piossasco (IT),Grand Ile (FR),190,,1000,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/21,7.468,44.9898,6.045,45.472 -IFA2,Tourbe (FR),Chilling (GB),,247.2,1000,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/25,-0.172042,49.083593,-1.277269,50.839338 -Italy-Montenegro,Villanova (IT),Latsva (MT),445,,1200,under construction,Link.14539,https://tyndp.entsoe.eu/tyndp2018/projects/projects/28,14.125,42.3947222222222,18.7947222222222,42.3175 -NordLink,Tonstad (NO),Wilster (DE),514,,1400,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/37,6.716948,58.662631,9.373979,53.922479 -COBRA cable,Endrup (DK),Eemshaven (NL),325,,700,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/71,8.718392,55.523115,6.835494,53.438589 -Thames Estuary Cluster (NEMO-Link),Richborough (GB),Gezelle (BE),140,,1000,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/74,1.324854,51.295891,3.23043,51.24902 -Anglo-Scottish -1,Hunterston (UK),Deeside (UK),422,,2400,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/77,-4.898329,55.723331,-3.032972,53.199735 +IFA2,Tourbe (FR),Chilling (GB),,247.2,1000,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/25,-0.172042,49.083593,-1.277269,50.839338 +Italy-Montenegro Phase 1,Villanova (IT),Latsva (MT),445,,600,built,Link.14539,https://tyndp.entsoe.eu/tyndp2018/projects/projects/28,14.125,42.3947222222222,18.7947222222222,42.3175 +Italy-Montenegro Phase 2,Villanova (IT),Latsva (MT),445,,600,under construction,Link.14539,https://tyndp.entsoe.eu/tyndp2018/projects/projects/28,14.125,42.3947222222222,18.7947222222222,42.3175 +NordLink,Tonstad (NO),Wilster (DE),514,,1400,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/37,6.716948,58.662631,9.373979,53.922479 +COBRA cable,Endrup (DK),Eemshaven (NL),325,,700,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/71,8.718392,55.523115,6.835494,53.438589 +Thames Estuary Cluster (NEMO-Link),Richborough (GB),Gezelle (BE),140,,1000,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/74,1.324854,51.295891,3.23043,51.24902 +Anglo-Scottish-1,Hunterston (UK),Deeside (UK),422,,2400,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/77,-4.898329,55.723331,-3.032972,53.199735 ALEGrO,Lixhe (BE),Oberzier (DE),100,,1000,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/92,5.67933,50.7567965,6.474704,50.867532 -North Sea Link,Kvilldal (NO),Blythe (GB),720,,1400,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/110,6.637527,59.515096,-1.510277,55.126957 +North Sea Link,Kvilldal (NO),Blythe (GB),720,,1400,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/110,6.637527,59.515096,-1.510277,55.126957 HVDC SuedOstLink,Wolmirstedt (DE),Isar (DE),,557,2000,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/130,11.629014,52.252137,12.091596,48.080837 HVDC Line A-North,Emden East (DE),Osterath (DE),,284,2000,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/132,7.206009,53.359403,6.619451,51.272935 France-Alderney-Britain,Exeter (UK),Menuel (FR),220,,1400,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/153,-3.533899,50.718412,-1.469216,49.509594 From 2a34ab26c4f09deb637ed3c143c8a6d96e7346a2 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 5 Apr 2022 16:07:04 +0200 Subject: [PATCH 22/54] add TYNDP link replaces --- data/links_tyndp.csv | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/data/links_tyndp.csv b/data/links_tyndp.csv index 38203979..dd55e514 100644 --- a/data/links_tyndp.csv +++ b/data/links_tyndp.csv @@ -2,14 +2,13 @@ Name,Converterstation 1,Converterstation 2,Length (given) (km),Length (distance* Biscay Gulf,Gatica (ES),Cubnezais (FR),370,,2200,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/16,-2.867,43.367,-0.408943,45.074191 Italy-France,Piossasco (IT),Grand Ile (FR),190,,1000,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/21,7.468,44.9898,6.045,45.472 IFA2,Tourbe (FR),Chilling (GB),,247.2,1000,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/25,-0.172042,49.083593,-1.277269,50.839338 -Italy-Montenegro Phase 1,Villanova (IT),Latsva (MT),445,,600,built,Link.14539,https://tyndp.entsoe.eu/tyndp2018/projects/projects/28,14.125,42.3947222222222,18.7947222222222,42.3175 -Italy-Montenegro Phase 2,Villanova (IT),Latsva (MT),445,,600,under construction,Link.14539,https://tyndp.entsoe.eu/tyndp2018/projects/projects/28,14.125,42.3947222222222,18.7947222222222,42.3175 -NordLink,Tonstad (NO),Wilster (DE),514,,1400,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/37,6.716948,58.662631,9.373979,53.922479 -COBRA cable,Endrup (DK),Eemshaven (NL),325,,700,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/71,8.718392,55.523115,6.835494,53.438589 +Italy-Montenegro,Villanova (IT),Latsva (MT),445,,1200,built,Link.14802,https://tyndp.entsoe.eu/tyndp2018/projects/projects/28,14.125,42.3947222222222,18.7947222222222,42.3175 +NordLink,Tonstad (NO),Wilster (DE),514,,1400,built,Link.14848,https://tyndp.entsoe.eu/tyndp2018/projects/projects/37,6.716948,58.662631,9.373979,53.922479 +COBRA cable,Endrup (DK),Eemshaven (NL),325,,700,built,Link.14803,https://tyndp.entsoe.eu/tyndp2018/projects/projects/71,8.718392,55.523115,6.835494,53.438589 Thames Estuary Cluster (NEMO-Link),Richborough (GB),Gezelle (BE),140,,1000,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/74,1.324854,51.295891,3.23043,51.24902 -Anglo-Scottish-1,Hunterston (UK),Deeside (UK),422,,2400,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/77,-4.898329,55.723331,-3.032972,53.199735 -ALEGrO,Lixhe (BE),Oberzier (DE),100,,1000,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/92,5.67933,50.7567965,6.474704,50.867532 -North Sea Link,Kvilldal (NO),Blythe (GB),720,,1400,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/110,6.637527,59.515096,-1.510277,55.126957 +Anglo-Scottish-1,Hunterston (UK),Deeside (UK),422,,2400,built,Link.8009,https://tyndp.entsoe.eu/tyndp2018/projects/projects/77,-4.898329,55.723331,-3.032972,53.199735 +ALEGrO,Lixhe (BE),Oberzier (DE),100,,1000,built,Link.14801,https://tyndp.entsoe.eu/tyndp2018/projects/projects/92,5.67933,50.7567965,6.474704,50.867532 +North Sea Link,Kvilldal (NO),Blythe (GB),720,,1400,built,Link.14804,https://tyndp.entsoe.eu/tyndp2018/projects/projects/110,6.637527,59.515096,-1.510277,55.126957 HVDC SuedOstLink,Wolmirstedt (DE),Isar (DE),,557,2000,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/130,11.629014,52.252137,12.091596,48.080837 HVDC Line A-North,Emden East (DE),Osterath (DE),,284,2000,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/132,7.206009,53.359403,6.619451,51.272935 France-Alderney-Britain,Exeter (UK),Menuel (FR),220,,1400,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/153,-3.533899,50.718412,-1.469216,49.509594 @@ -24,4 +23,4 @@ HVDC Ultranet,Osterath (DE),Philippsburg (DE),,314,600,in permitting,,https://ty Gridlink,Kingsnorth (UK),Warande (FR),160,,1400,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/285,0.596111111111111,51.41972,2.376776,51.034368 NeuConnect,Grain (UK),Fedderwarden (DE),680,,1400,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/309,0.716666666666667,51.44,8.046524,53.562763 NordBalt,Klaipeda (LT),Nybro (SE),450,,700,built,,https://en.wikipedia.org/wiki/NordBalt,21.256667,55.681667,15.854167,56.767778 -Estlink 1,Harku (EE),Espoo (FI),105,,350,built,,https://en.wikipedia.org/wiki/Estlink,24.560278,59.384722,24.551667,60.203889 +Estlink 1,Harku (EE),Espoo (FI),105,,350,built,Link.14807,https://en.wikipedia.org/wiki/Estlink,24.560278,59.384722,24.551667,60.203889 From eca4a017dbb882645b49eb6114ff9731e4b476ce Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 5 Apr 2022 16:07:30 +0200 Subject: [PATCH 23/54] implement TYNDP link overriding for link_id not oid --- scripts/base_network.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/base_network.py b/scripts/base_network.py index 50ec8e53..646a3e9c 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -196,14 +196,15 @@ def _add_links_from_tyndp(buses, links, links_tyndp, europe_shape): return buses, links has_replaces_b = links_tyndp.replaces.notnull() - oids = dict(Bus=_get_oid(buses), Link=_get_oid(links)) + logger.info("TYNDP links replacing links in dataset (overwriting): " + ", ".join(links_tyndp.loc[has_replaces_b, "Name"])) + ids = dict(Bus=buses.index, Link=links.index) keep_b = dict(Bus=pd.Series(True, index=buses.index), Link=pd.Series(True, index=links.index)) for reps in links_tyndp.loc[has_replaces_b, 'replaces']: for comps in reps.split(':'): - oids_to_remove = comps.split('.') - c = oids_to_remove.pop(0) - keep_b[c] &= ~oids[c].isin(oids_to_remove) + ids_to_remove = comps.split('.') + c = ids_to_remove.pop(0) + keep_b[c] &= ~ids[c].isin(ids_to_remove) buses = buses.loc[keep_b['Bus']] links = links.loc[keep_b['Link']] @@ -211,7 +212,7 @@ def _add_links_from_tyndp(buses, links, links_tyndp, europe_shape): # Corresponds approximately to 20km tolerances if links_tyndp["j"].notnull().any(): - logger.info("TYNDP links already in the dataset (skipping): " + ", ".join(links_tyndp.loc[links_tyndp["j"].notnull(), "Name"])) + logger.info("Additional TYNDP links already in the dataset (skipping): " + ", ".join(links_tyndp.loc[links_tyndp["j"].notnull(), "Name"])) links_tyndp = links_tyndp.loc[links_tyndp["j"].isnull()] if links_tyndp.empty: return buses, links From aa6e98a56375b6c8c2c16cfc541a48ca4b239277 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 5 Apr 2022 16:11:02 +0200 Subject: [PATCH 24/54] label built TYNDP links as not under_construction --- scripts/base_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/base_network.py b/scripts/base_network.py index 646a3e9c..9e8421ab 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -236,7 +236,7 @@ def _add_links_from_tyndp(buses, links, links_tyndp, europe_shape): carrier='DC', p_nom=links_tyndp["Power (MW)"], length=links_tyndp["Length (given) (km)"].fillna(links_tyndp["Length (distance*1.2) (km)"]), - under_construction=True, + under_construction=~(links_tyndp.status == 'built'), underground=False, geometry=(links_tyndp[["x1", "y1", "x2", "y2"]] .apply(lambda s: str(LineString([[s.x1, s.y1], [s.x2, s.y2]])), axis=1)), From bfcc20aa13578354e90fe851f4761dd70e726ba9 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 5 Apr 2022 17:10:12 +0200 Subject: [PATCH 25/54] Revert "update TYNDP links which are already built" --- data/links_tyndp.csv | 18 +++++++++--------- scripts/base_network.py | 13 ++++++------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/data/links_tyndp.csv b/data/links_tyndp.csv index dd55e514..8079be72 100644 --- a/data/links_tyndp.csv +++ b/data/links_tyndp.csv @@ -1,14 +1,14 @@ Name,Converterstation 1,Converterstation 2,Length (given) (km),Length (distance*1.2) (km),Power (MW),status,replaces,Ref,x1,y1,x2,y2 Biscay Gulf,Gatica (ES),Cubnezais (FR),370,,2200,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/16,-2.867,43.367,-0.408943,45.074191 Italy-France,Piossasco (IT),Grand Ile (FR),190,,1000,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/21,7.468,44.9898,6.045,45.472 -IFA2,Tourbe (FR),Chilling (GB),,247.2,1000,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/25,-0.172042,49.083593,-1.277269,50.839338 -Italy-Montenegro,Villanova (IT),Latsva (MT),445,,1200,built,Link.14802,https://tyndp.entsoe.eu/tyndp2018/projects/projects/28,14.125,42.3947222222222,18.7947222222222,42.3175 -NordLink,Tonstad (NO),Wilster (DE),514,,1400,built,Link.14848,https://tyndp.entsoe.eu/tyndp2018/projects/projects/37,6.716948,58.662631,9.373979,53.922479 -COBRA cable,Endrup (DK),Eemshaven (NL),325,,700,built,Link.14803,https://tyndp.entsoe.eu/tyndp2018/projects/projects/71,8.718392,55.523115,6.835494,53.438589 -Thames Estuary Cluster (NEMO-Link),Richborough (GB),Gezelle (BE),140,,1000,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/74,1.324854,51.295891,3.23043,51.24902 -Anglo-Scottish-1,Hunterston (UK),Deeside (UK),422,,2400,built,Link.8009,https://tyndp.entsoe.eu/tyndp2018/projects/projects/77,-4.898329,55.723331,-3.032972,53.199735 -ALEGrO,Lixhe (BE),Oberzier (DE),100,,1000,built,Link.14801,https://tyndp.entsoe.eu/tyndp2018/projects/projects/92,5.67933,50.7567965,6.474704,50.867532 -North Sea Link,Kvilldal (NO),Blythe (GB),720,,1400,built,Link.14804,https://tyndp.entsoe.eu/tyndp2018/projects/projects/110,6.637527,59.515096,-1.510277,55.126957 +IFA2,Tourbe (FR),Chilling (GB),,247.2,1000,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/25,-0.172042,49.083593,-1.277269,50.839338 +Italy-Montenegro,Villanova (IT),Latsva (MT),445,,1200,under construction,Link.14539,https://tyndp.entsoe.eu/tyndp2018/projects/projects/28,14.125,42.3947222222222,18.7947222222222,42.3175 +NordLink,Tonstad (NO),Wilster (DE),514,,1400,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/37,6.716948,58.662631,9.373979,53.922479 +COBRA cable,Endrup (DK),Eemshaven (NL),325,,700,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/71,8.718392,55.523115,6.835494,53.438589 +Thames Estuary Cluster (NEMO-Link),Richborough (GB),Gezelle (BE),140,,1000,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/74,1.324854,51.295891,3.23043,51.24902 +Anglo-Scottish -1,Hunterston (UK),Deeside (UK),422,,2400,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/77,-4.898329,55.723331,-3.032972,53.199735 +ALEGrO,Lixhe (BE),Oberzier (DE),100,,1000,built,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/92,5.67933,50.7567965,6.474704,50.867532 +North Sea Link,Kvilldal (NO),Blythe (GB),720,,1400,under construction,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/110,6.637527,59.515096,-1.510277,55.126957 HVDC SuedOstLink,Wolmirstedt (DE),Isar (DE),,557,2000,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/130,11.629014,52.252137,12.091596,48.080837 HVDC Line A-North,Emden East (DE),Osterath (DE),,284,2000,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/132,7.206009,53.359403,6.619451,51.272935 France-Alderney-Britain,Exeter (UK),Menuel (FR),220,,1400,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/153,-3.533899,50.718412,-1.469216,49.509594 @@ -23,4 +23,4 @@ HVDC Ultranet,Osterath (DE),Philippsburg (DE),,314,600,in permitting,,https://ty Gridlink,Kingsnorth (UK),Warande (FR),160,,1400,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/285,0.596111111111111,51.41972,2.376776,51.034368 NeuConnect,Grain (UK),Fedderwarden (DE),680,,1400,in permitting,,https://tyndp.entsoe.eu/tyndp2018/projects/projects/309,0.716666666666667,51.44,8.046524,53.562763 NordBalt,Klaipeda (LT),Nybro (SE),450,,700,built,,https://en.wikipedia.org/wiki/NordBalt,21.256667,55.681667,15.854167,56.767778 -Estlink 1,Harku (EE),Espoo (FI),105,,350,built,Link.14807,https://en.wikipedia.org/wiki/Estlink,24.560278,59.384722,24.551667,60.203889 +Estlink 1,Harku (EE),Espoo (FI),105,,350,built,,https://en.wikipedia.org/wiki/Estlink,24.560278,59.384722,24.551667,60.203889 diff --git a/scripts/base_network.py b/scripts/base_network.py index 9e8421ab..50ec8e53 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -196,15 +196,14 @@ def _add_links_from_tyndp(buses, links, links_tyndp, europe_shape): return buses, links has_replaces_b = links_tyndp.replaces.notnull() - logger.info("TYNDP links replacing links in dataset (overwriting): " + ", ".join(links_tyndp.loc[has_replaces_b, "Name"])) - ids = dict(Bus=buses.index, Link=links.index) + oids = dict(Bus=_get_oid(buses), Link=_get_oid(links)) keep_b = dict(Bus=pd.Series(True, index=buses.index), Link=pd.Series(True, index=links.index)) for reps in links_tyndp.loc[has_replaces_b, 'replaces']: for comps in reps.split(':'): - ids_to_remove = comps.split('.') - c = ids_to_remove.pop(0) - keep_b[c] &= ~ids[c].isin(ids_to_remove) + oids_to_remove = comps.split('.') + c = oids_to_remove.pop(0) + keep_b[c] &= ~oids[c].isin(oids_to_remove) buses = buses.loc[keep_b['Bus']] links = links.loc[keep_b['Link']] @@ -212,7 +211,7 @@ def _add_links_from_tyndp(buses, links, links_tyndp, europe_shape): # Corresponds approximately to 20km tolerances if links_tyndp["j"].notnull().any(): - logger.info("Additional TYNDP links already in the dataset (skipping): " + ", ".join(links_tyndp.loc[links_tyndp["j"].notnull(), "Name"])) + logger.info("TYNDP links already in the dataset (skipping): " + ", ".join(links_tyndp.loc[links_tyndp["j"].notnull(), "Name"])) links_tyndp = links_tyndp.loc[links_tyndp["j"].isnull()] if links_tyndp.empty: return buses, links @@ -236,7 +235,7 @@ def _add_links_from_tyndp(buses, links, links_tyndp, europe_shape): carrier='DC', p_nom=links_tyndp["Power (MW)"], length=links_tyndp["Length (given) (km)"].fillna(links_tyndp["Length (distance*1.2) (km)"]), - under_construction=~(links_tyndp.status == 'built'), + under_construction=True, underground=False, geometry=(links_tyndp[["x1", "y1", "x2", "y2"]] .apply(lambda s: str(LineString([[s.x1, s.y1], [s.x2, s.y2]])), axis=1)), From 3acc4a22360cbf6c3f22c63880f55c465ee8244b Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 12 Apr 2022 14:52:37 +0200 Subject: [PATCH 26/54] update country reference from Germany to Belgium in tutorial --- doc/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorial.rst b/doc/tutorial.rst index 76970aff..c37abb39 100644 --- a/doc/tutorial.rst +++ b/doc/tutorial.rst @@ -43,7 +43,7 @@ For more information on the data dependencies of PyPSA-Eur, continue reading :re How to customise PyPSA-Eur? =========================== -The model can be adapted to only include selected countries (e.g. Germany) instead of all European countries to limit the spatial scope. +The model can be adapted to only include selected countries (e.g. Belgium) instead of all European countries to limit the spatial scope. .. literalinclude:: ../config.tutorial.yaml :language: yaml From 49f1c0f6299401fc99ea2ebb849ff45be68397a4 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 15 Apr 2022 18:36:37 +0200 Subject: [PATCH 27/54] Merge pull request #348 from PyPSA/annuity add_electricity: remove vresutils.costdata.annuity dependency --- scripts/add_electricity.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index ad932cd8..ceef2390 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -94,7 +94,6 @@ import geopandas as gpd import powerplantmatching as pm from powerplantmatching.export import map_country_bus -from vresutils.costdata import annuity from vresutils import transfer as vtransfer idx = pd.IndexSlice @@ -105,6 +104,18 @@ logger = logging.getLogger(__name__) def normed(s): return s/s.sum() +def calculate_annuity(n, r): + """Calculate the annuity factor for an asset with lifetime n years and + discount rate of r, e.g. annuity(20, 0.05) * 20 = 1.6""" + + if isinstance(r, pd.Series): + return pd.Series(1/n, index=r.index).where(r == 0, r/(1. - 1./(1.+r)**n)) + elif r > 0: + return r / (1. - 1./(1.+r)**n) + else: + return 1 / n + + def _add_missing_carriers_from_costs(n, costs, carriers): missing_carriers = pd.Index(carriers).difference(n.carriers.index) if missing_carriers.empty: return @@ -138,7 +149,7 @@ def load_costs(tech_costs, config, elec_config, Nyears=1.): "investment" : 0, "lifetime" : 25}) - costs["capital_cost"] = ((annuity(costs["lifetime"], costs["discount rate"]) + + costs["capital_cost"] = ((calculate_annuity(costs["lifetime"], costs["discount rate"]) + costs["FOM"]/100.) * costs["investment"] * Nyears) From dcac3ea6e1f8df9c8eca4e67c87512412310d252 Mon Sep 17 00:00:00 2001 From: martacki Date: Thu, 28 Apr 2022 12:59:25 +0200 Subject: [PATCH 28/54] respect stores in make_summary script --- scripts/make_summary.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/make_summary.py b/scripts/make_summary.py index af1ecf36..a14000ef 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -171,6 +171,9 @@ def calculate_capacity(n,label,capacity): if 'p_nom_opt' in c.df.columns: c_capacities = abs(c.df.p_nom_opt.multiply(c.df.sign)).groupby(c.df.carrier).sum() capacity = include_in_summary(capacity, [c.list_name], label, c_capacities) + elif 'e_nom_opt' in c.df.columns: + c_capacities = abs(c.df.e_nom_opt.multiply(c.df.sign)).groupby(c.df.carrier).sum() + capacity = include_in_summary(capacity, [c.list_name], label, c_capacities) for c in n.iterate_components(n.passive_branch_components): c_capacities = c.df['s_nom_opt'].groupby(c.df.carrier).sum() From d12405d848ce83134bf0011f25d2394c020733d4 Mon Sep 17 00:00:00 2001 From: martacki Date: Fri, 29 Apr 2022 13:48:58 +0200 Subject: [PATCH 29/54] respect stores for energy_supply.csv --- scripts/make_summary.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/make_summary.py b/scripts/make_summary.py index a14000ef..972b245d 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -235,11 +235,11 @@ def calculate_supply(n, label, supply): def calculate_supply_energy(n, label, supply_energy): """calculate the total dispatch of each component at the buses where the loads are attached""" - load_types = n.loads.carrier.value_counts().index + load_types = n.buses.carrier.unique() for i in load_types: - buses = n.loads.bus[n.loads.carrier == i].values + buses = n.buses.query("carrier == @i").index bus_map = pd.Series(False,index=n.buses.index) From ac7c94337e98735906d87bb2da5c6b1d4bc2a5f2 Mon Sep 17 00:00:00 2001 From: martacki Date: Fri, 29 Apr 2022 13:51:54 +0200 Subject: [PATCH 30/54] respect stores for supply.csv --- scripts/make_summary.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 972b245d..c070d33f 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -188,11 +188,11 @@ def calculate_capacity(n,label,capacity): def calculate_supply(n, label, supply): """calculate the max dispatch of each component at the buses where the loads are attached""" - load_types = n.loads.carrier.value_counts().index + load_types = n.buses.carrier.unique() for i in load_types: - buses = n.loads.bus[n.loads.carrier == i].values + buses = n.buses.query("carrier == @i").index bus_map = pd.Series(False,index=n.buses.index) From 8a49697a51ee4148a565d2937cdd69d9a761810b Mon Sep 17 00:00:00 2001 From: martacki Date: Fri, 29 Apr 2022 15:09:10 +0200 Subject: [PATCH 31/54] bugfixes for manual load adjustments --- config.default.yaml | 2 +- config.tutorial.yaml | 2 +- scripts/build_load_data.py | 24 ++++++++++++++++++------ 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index d2bf6159..8ac51ee8 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -26,7 +26,7 @@ clustering: snapshots: start: "2013-01-01" end: "2014-01-01" - closed: 'left' # end is not inclusive + inclusive: 'left' # end is not inclusive enable: prepare_links_p_nom: false diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 225d8f78..2ac96ee3 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -26,7 +26,7 @@ clustering: snapshots: start: "2013-03-01" end: "2013-04-01" - closed: 'left' # end is not inclusive + inclusive: 'left' # end is not inclusive enable: prepare_links_p_nom: false diff --git a/scripts/build_load_data.py b/scripts/build_load_data.py index 10921782..3c11b947 100755 --- a/scripts/build_load_data.py +++ b/scripts/build_load_data.py @@ -116,14 +116,20 @@ def nan_statistics(df): keys=['total', 'consecutive', 'max_total_per_month'], axis=1) -def copy_timeslice(load, cntry, start, stop, delta): +def copy_timeslice(load, cntry, start, stop, delta, fn_load=None): start = pd.Timestamp(start) stop = pd.Timestamp(stop) - if start-delta in load.index and stop in load.index and cntry in load: - load.loc[start:stop, cntry] = load.loc[start-delta:stop-delta, cntry].values + if (start in load.index and stop in load.index): + if start-delta in load.index and stop-delta in load.index and cntry in load: + load.loc[start:stop, cntry] = load.loc[start-delta:stop-delta, cntry].values + elif fn_load is not None: + duration = pd.date_range(freq='h', start=start-delta, end=stop-delta) + load_raw = load_timeseries(fn_load, duration, [cntry], powerstatistics) + if start-delta in load_raw.index and stop-delta in load_raw.index and cntry in load_raw: + load.loc[start:stop, cntry] = load_raw.loc[start-delta:stop-delta, cntry].values -def manual_adjustment(load, powerstatistics): +def manual_adjustment(load, fn_load, powerstatistics): """ Adjust gaps manual for load data from OPSD time-series package. @@ -150,6 +156,8 @@ def manual_adjustment(load, powerstatistics): powerstatistics: bool Whether argument load comprises the electricity consumption data of the ENTSOE power statistics or of the ENTSOE transparency map + load_fn: str + File name or url location (file format .csv) Returns ------- @@ -175,7 +183,11 @@ def manual_adjustment(load, powerstatistics): copy_timeslice(load, 'CH', '2010-11-04 04:00', '2010-11-04 22:00', Delta(days=1)) copy_timeslice(load, 'NO', '2010-12-09 11:00', '2010-12-09 18:00', Delta(days=1)) # whole january missing - copy_timeslice(load, 'GB', '2009-12-31 23:00', '2010-01-31 23:00', Delta(days=-364)) + copy_timeslice(load, 'GB', '2010-01-01 00:00', '2010-01-31 23:00', Delta(days=-365), fn_load) + # 1.1. at midnight gets special treatment + copy_timeslice(load, 'IE', '2016-01-01 00:00', '2016-01-01 01:00', Delta(days=-366), fn_load) + copy_timeslice(load, 'PT', '2016-01-01 00:00', '2016-01-01 01:00', Delta(days=-366), fn_load) + copy_timeslice(load, 'GB', '2016-01-01 00:00', '2016-01-01 01:00', Delta(days=-366), fn_load) else: if 'ME' in load: @@ -206,7 +218,7 @@ if __name__ == "__main__": load = load_timeseries(snakemake.input[0], years, countries, powerstatistics) if snakemake.config['load']['manual_adjustments']: - load = manual_adjustment(load, powerstatistics) + load = manual_adjustment(load, snakemake.input[0], powerstatistics) logger.info(f"Linearly interpolate gaps of size {interpolate_limit} and less.") load = load.interpolate(method='linear', limit=interpolate_limit) From 2e4c30e28e7a0d5a774116a77cdf87d9750f48aa Mon Sep 17 00:00:00 2001 From: martacki Date: Fri, 29 Apr 2022 15:18:22 +0200 Subject: [PATCH 32/54] abbreviate --- scripts/build_load_data.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/build_load_data.py b/scripts/build_load_data.py index 3c11b947..55270e49 100755 --- a/scripts/build_load_data.py +++ b/scripts/build_load_data.py @@ -125,8 +125,7 @@ def copy_timeslice(load, cntry, start, stop, delta, fn_load=None): elif fn_load is not None: duration = pd.date_range(freq='h', start=start-delta, end=stop-delta) load_raw = load_timeseries(fn_load, duration, [cntry], powerstatistics) - if start-delta in load_raw.index and stop-delta in load_raw.index and cntry in load_raw: - load.loc[start:stop, cntry] = load_raw.loc[start-delta:stop-delta, cntry].values + load.loc[start:stop, cntry] = load_raw.loc[start-delta:stop-delta, cntry].values def manual_adjustment(load, fn_load, powerstatistics): From fa0a028499ba7113c10b3fa74ea4726f11825869 Mon Sep 17 00:00:00 2001 From: martacki Date: Mon, 2 May 2022 10:01:51 +0200 Subject: [PATCH 33/54] pandas compatibility --- config.default.yaml | 2 +- config.tutorial.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index 8ac51ee8..d2bf6159 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -26,7 +26,7 @@ clustering: snapshots: start: "2013-01-01" end: "2014-01-01" - inclusive: 'left' # end is not inclusive + closed: 'left' # end is not inclusive enable: prepare_links_p_nom: false diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 2ac96ee3..225d8f78 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -26,7 +26,7 @@ clustering: snapshots: start: "2013-03-01" end: "2013-04-01" - inclusive: 'left' # end is not inclusive + closed: 'left' # end is not inclusive enable: prepare_links_p_nom: false From 4706422f4b5e2da52031d59440909374cec35be6 Mon Sep 17 00:00:00 2001 From: Ebbe Kyhl <69363603+ebbekyhl@users.noreply.github.com> Date: Fri, 27 May 2022 16:14:01 +0200 Subject: [PATCH 34/54] Update version of powerplantmatching Hi, I recently became aware that I was using an older version (0.4.8) of the powerplantmatching. I tested my setup with the newer version (0.5.3), and it runs without any issues. The following is a comment/question on the powerplantmatching dataset, which maybe is relevant to mention: With regard to pumped-hydro storage (PHS), the newest version of powerplantmatching entails an energy storage capacity of 4.3 TWh (Europe-aggregate, assuming 6-hours duration for plants that do not have duration specified). In the earlier version 0.4.8, this was vastly overestimated at 10 TWh. As comparison, Geth et al. (2015) showed 1.3 TWh (including Norway and Switzerland) using 2012-numbers. PHS power capacity has increased from roughly 50 to 55 GW from 2014 to 2020 (iha, 2015, 2021), so energy storage capacity most likely is increased as well. But is it fair to say that energy storage capacity has been quadrupled since 2012 (from 1.3 TWh to 4.3 TWh)? Or how can we interpret this difference? Sources: Geth et al., 2015, https://doi.org/10.1016/j.rser.2015.07.145 iha, 2015, https://www.aler-renovaveis.org/contents/lerpublication/iha_2015_sept_hydropower-status-report.pdf iha, 2021, https://assets-global.website-files.com/5f749e4b9399c80b5e421384/60c37321987070812596e26a_IHA20212405-status-report-02_LR.pdf --- envs/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 3c69b77b..eaad600f 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -24,7 +24,7 @@ dependencies: - yaml - pytables - lxml - - powerplantmatching>=0.4.8 + - powerplantmatching==0.5.3 - numpy - pandas - geopandas From b6032fb891c3506d1720b0609fa979e6d2d87321 Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Sat, 28 May 2022 14:48:32 +0100 Subject: [PATCH 35/54] fix crs bug --- scripts/build_renewable_profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index a2b2eda6..41e1b54d 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -240,7 +240,7 @@ if __name__ == '__main__': # use named function np.greater with partially frozen argument instead # and exclude areas where: -max_depth > grid cell depth func = functools.partial(np.greater,-config['max_depth']) - excluder.add_raster(snakemake.input.gebco, codes=func, crs=4236, nodata=-1000) + excluder.add_raster(snakemake.input.gebco, codes=func, crs=4326, nodata=-1000) if 'min_shore_distance' in config: buffer = config['min_shore_distance'] From edb81a9e6a4ce2f7f53aeacf68d15aaa48799509 Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Sat, 28 May 2022 14:50:44 +0100 Subject: [PATCH 36/54] add release note --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 3f131dc0..80211635 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -70,6 +70,8 @@ Upcoming Release * Use updated SARAH-2 and ERA5 cutouts with slightly wider scope to east and additional variables. +* Fix crs bug. Change crs 4236 to 4326. + PyPSA-Eur 0.4.0 (22th September 2021) ===================================== From 3bb8a7967a92b15f86ad5d914be1ada37bf60ff2 Mon Sep 17 00:00:00 2001 From: Ebbe Kyhl <69363603+ebbekyhl@users.noreply.github.com> Date: Mon, 30 May 2022 08:15:54 +0200 Subject: [PATCH 37/54] Powerplantmatching version 0.5.3 as lower bound --- envs/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index eaad600f..a2cba37f 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -24,7 +24,7 @@ dependencies: - yaml - pytables - lxml - - powerplantmatching==0.5.3 + - powerplantmatching>=0.5.3 - numpy - pandas - geopandas From 798c015bf6e8f34e35d5e9accd5eb0323487c655 Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Fri, 3 Jun 2022 17:03:10 +0100 Subject: [PATCH 38/54] restrict rasterio version --- envs/environment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index a2cba37f..69025845 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -37,7 +37,7 @@ dependencies: - pyomo - matplotlib - proj - - fiona <= 1.18.20 # Till issue https://github.com/Toblerity/Fiona/issues/1085 is not solved + - fiona<=1.18.20 # Till issue https://github.com/Toblerity/Fiona/issues/1085 is not solved # Keep in conda environment when calling ipython - ipython @@ -45,7 +45,7 @@ dependencies: # GIS dependencies: - cartopy - descartes - - rasterio + - rasterio<=1.2.8 # 1.2.10 creates error https://github.com/PyPSA/atlite/issues/238 # PyPSA-Eur-Sec Dependencies - geopy From 21f627c74ec0b3153d60969f542052694b142e1c Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Fri, 3 Jun 2022 17:05:32 +0100 Subject: [PATCH 39/54] update version --- envs/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 69025845..f8060de1 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -45,7 +45,7 @@ dependencies: # GIS dependencies: - cartopy - descartes - - rasterio<=1.2.8 # 1.2.10 creates error https://github.com/PyPSA/atlite/issues/238 + - rasterio<=1.2.9 # 1.2.10 creates error https://github.com/PyPSA/atlite/issues/238 # PyPSA-Eur-Sec Dependencies - geopy From e974a30fd3dcf6cc8d005251b1247778cdffb933 Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Fri, 3 Jun 2022 17:14:30 +0100 Subject: [PATCH 40/54] add release note --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 80211635..963a1175 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -72,6 +72,8 @@ Upcoming Release * Fix crs bug. Change crs 4236 to 4326. +* Update rasterio version to correctly calculate exclusion raster + PyPSA-Eur 0.4.0 (22th September 2021) ===================================== From d5db3b8d8060dfd7d4a33c5749a4c2c70ef64864 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 7 Jun 2022 10:57:01 +0200 Subject: [PATCH 41/54] Update scripts/build_bus_regions.py Co-authored-by: Fabian Hofmann --- scripts/build_bus_regions.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index 4f2369b6..382a32e8 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -117,9 +117,7 @@ def voronoi_partition_pts(points, outline, no_multipolygons=False): return poly polygons = [demultipolygon(poly) for poly in polygons] - polygons_arr = np.empty((len(polygons),), 'object') - polygons_arr[:] = polygons - return polygons_arr + return np.array(polygons, dtype=object) if __name__ == "__main__": From aa867cb70489041cd7fdbed61ebec85a52e38ed6 Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Tue, 7 Jun 2022 15:00:57 +0200 Subject: [PATCH 42/54] Update scripts/build_bus_regions.py Co-authored-by: Fabian Neumann --- scripts/build_bus_regions.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index 382a32e8..89765ed9 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -107,15 +107,6 @@ def voronoi_partition_pts(points, outline, no_multipolygons=False): polygons.append(poly) - if no_multipolygons: - def demultipolygon(poly): - try: - # for a MultiPolygon pick the part with the largest area - poly = max(poly.geoms, key=lambda pg: pg.area) - except: - pass - return poly - polygons = [demultipolygon(poly) for poly in polygons] return np.array(polygons, dtype=object) From 97fbf77ff8ddb7da8bb75602b73853e319854cb5 Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Tue, 7 Jun 2022 15:01:18 +0200 Subject: [PATCH 43/54] Update scripts/build_bus_regions.py --- scripts/build_bus_regions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index 89765ed9..b6d2d129 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -62,7 +62,7 @@ def save_to_geojson(s, fn): s.to_file(fn, driver='GeoJSON', schema=schema) -def voronoi_partition_pts(points, outline, no_multipolygons=False): +def voronoi_partition_pts(points, outline): """ Compute the polygons of a voronoi partition of `points` within the polygon `outline`. Taken from From bdd094d796b3f896b42ba8d6fb27c0093f467a67 Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Tue, 7 Jun 2022 15:01:40 +0200 Subject: [PATCH 44/54] Update scripts/build_bus_regions.py --- scripts/build_bus_regions.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index b6d2d129..8003d370 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -71,8 +71,6 @@ def voronoi_partition_pts(points, outline): ---------- points : Nx2 - ndarray[dtype=float] outline : Polygon - no_multipolygons : bool (default: False) - If true, replace each MultiPolygon by its largest component Returns ------- polygons : N - ndarray[dtype=Polygon|MultiPolygon] From cd92d8092ba2ba332e5c8e0ea3b29ff78e89c2b8 Mon Sep 17 00:00:00 2001 From: martacki Date: Wed, 8 Jun 2022 15:49:06 +0200 Subject: [PATCH 45/54] plot_summary: remove deprecated retrieve_snakemake_keys function --- scripts/plot_summary.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 48f064b0..bc2bd30c 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -21,7 +21,7 @@ Description import os import logging -from _helpers import configure_logging, retrieve_snakemake_keys +from _helpers import configure_logging import pandas as pd import matplotlib.pyplot as plt @@ -170,12 +170,12 @@ if __name__ == "__main__": attr='', ext='png', country='all') configure_logging(snakemake) - paths, config, wildcards, logs, out = retrieve_snakemake_keys(snakemake) + config = snakemake.config - summary = wildcards.summary + summary = snakemake.wildcards.summary try: func = globals()[f"plot_{summary}"] except KeyError: raise RuntimeError(f"plotting function for {summary} has not been defined") - func(os.path.join(paths[0], f"{summary}.csv"), config, out[0]) + func(os.path.join(snakemake.input[0], f"{summary}.csv"), config, snakemake.output[0]) From 1a7b439f2d43bd24f685e3602d6be6d122d557b1 Mon Sep 17 00:00:00 2001 From: zoltanmaric Date: Wed, 8 Jun 2022 17:19:06 +0200 Subject: [PATCH 46/54] Remove usages of `retrieve_snakemake_keys` --- scripts/build_natura_raster.py | 11 ++++------- scripts/plot_p_nom_max.py | 14 ++++++-------- scripts/prepare_links_p_nom.py | 6 ++---- 3 files changed, 12 insertions(+), 19 deletions(-) diff --git a/scripts/build_natura_raster.py b/scripts/build_natura_raster.py index 71d2c45e..7fa9d544 100644 --- a/scripts/build_natura_raster.py +++ b/scripts/build_natura_raster.py @@ -40,7 +40,7 @@ Description """ import logging -from _helpers import configure_logging, retrieve_snakemake_keys +from _helpers import configure_logging import atlite import geopandas as gpd @@ -73,20 +73,17 @@ if __name__ == "__main__": snakemake = mock_snakemake('build_natura_raster') configure_logging(snakemake) - paths, config, wildcards, logs, out = retrieve_snakemake_keys(snakemake) - - cutouts = paths.cutouts + cutouts = snakemake.input.cutouts xs, Xs, ys, Ys = zip(*(determine_cutout_xXyY(cutout) for cutout in cutouts)) bounds = transform_bounds(4326, 3035, min(xs), min(ys), max(Xs), max(Ys)) transform, out_shape = get_transform_and_shape(bounds, res=100) # adjusted boundaries - shapes = gpd.read_file(paths.natura).to_crs(3035) + shapes = gpd.read_file(snakemake.input.natura).to_crs(3035) raster = ~geometry_mask(shapes.geometry, out_shape[::-1], transform) raster = raster.astype(rio.uint8) - with rio.open(out[0], 'w', driver='GTiff', dtype=rio.uint8, + with rio.open(snakemake.output[0], 'w', driver='GTiff', dtype=rio.uint8, count=1, transform=transform, crs=3035, compress='lzw', width=raster.shape[1], height=raster.shape[0]) as dst: dst.write(raster, indexes=1) - diff --git a/scripts/plot_p_nom_max.py b/scripts/plot_p_nom_max.py index ea66d612..e79ad274 100644 --- a/scripts/plot_p_nom_max.py +++ b/scripts/plot_p_nom_max.py @@ -19,7 +19,7 @@ Description """ import logging -from _helpers import configure_logging, retrieve_snakemake_keys +from _helpers import configure_logging import pypsa import pandas as pd @@ -53,13 +53,11 @@ if __name__ == "__main__": clusts= '5,full', country= 'all') configure_logging(snakemake) - paths, config, wildcards, logs, out = retrieve_snakemake_keys(snakemake) - plot_kwds = dict(drawstyle="steps-post") - clusters = wildcards.clusts.split(',') - techs = wildcards.techs.split(',') - country = wildcards.country + clusters = snakemake.wildcards.clusts.split(',') + techs = snakemake.wildcards.techs.split(',') + country = snakemake.wildcards.country if country == 'all': country = None else: @@ -68,7 +66,7 @@ if __name__ == "__main__": fig, axes = plt.subplots(1, len(techs)) for j, cluster in enumerate(clusters): - net = pypsa.Network(paths[j]) + net = pypsa.Network(snakemake.input[j]) for i, tech in enumerate(techs): cum_p_nom_max(net, tech, country).plot(x="p_max_pu", y="cum_p_nom_max", @@ -81,4 +79,4 @@ if __name__ == "__main__": plt.legend(title="Cluster level") - fig.savefig(out[0], transparent=True, bbox_inches='tight') + fig.savefig(snakemake.output[0], transparent=True, bbox_inches='tight') diff --git a/scripts/prepare_links_p_nom.py b/scripts/prepare_links_p_nom.py index 6bd4bca4..b83089d6 100644 --- a/scripts/prepare_links_p_nom.py +++ b/scripts/prepare_links_p_nom.py @@ -37,7 +37,7 @@ Description """ import logging -from _helpers import configure_logging, retrieve_snakemake_keys +from _helpers import configure_logging import pandas as pd @@ -63,8 +63,6 @@ if __name__ == "__main__": snakemake = mock_snakemake('prepare_links_p_nom', simpl='', network='elec') configure_logging(snakemake) - paths, config, wildcards, logs, out = retrieve_snakemake_keys(snakemake) - links_p_nom = pd.read_html('https://en.wikipedia.org/wiki/List_of_HVDC_projects', header=0, match="SwePol")[0] mw = "Power (MW)" @@ -76,4 +74,4 @@ if __name__ == "__main__": links_p_nom['x1'], links_p_nom['y1'] = extract_coordinates(links_p_nom['Converterstation 1']) links_p_nom['x2'], links_p_nom['y2'] = extract_coordinates(links_p_nom['Converterstation 2']) - links_p_nom.dropna(subset=['x1', 'y1', 'x2', 'y2']).to_csv(out[0], index=False) + links_p_nom.dropna(subset=['x1', 'y1', 'x2', 'y2']).to_csv(snakemake.output[0], index=False) From 6b9932f5e80d685579617f2d962b4f9e77763263 Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Tue, 14 Jun 2022 15:24:10 +0200 Subject: [PATCH 47/54] build_renewable_profiles: set show progress default to False --- scripts/build_renewable_profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 41e1b54d..70cadab4 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -203,7 +203,7 @@ if __name__ == '__main__': pgb.streams.wrap_stderr() nprocesses = int(snakemake.threads) - noprogress = not snakemake.config['atlite'].get('show_progress', True) + noprogress = not snakemake.config['atlite'].get('show_progress', False) config = snakemake.config['renewable'][snakemake.wildcards.technology] resource = config['resource'] # pv panel config / wind turbine config correction_factor = config.get('correction_factor', 1.) From 5df588ccb80026eaf60b2d70eb71f3c8c55923bc Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Mon, 20 Jun 2022 12:45:28 +0200 Subject: [PATCH 48/54] fix snakemake error introduced after v7.7.0 --- scripts/_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 410e05af..766fb421 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -240,7 +240,7 @@ def mock_snakemake(rulename, **wildcards): if os.path.exists(p): snakefile = p break - workflow = sm.Workflow(snakefile, overwrite_configfiles=[]) + workflow = sm.Workflow(snakefile, overwrite_configfiles=[], rerun_triggers=[]) workflow.include(snakefile) workflow.global_resources = {} rule = workflow.get_rule(rulename) From c2413aeef439ffb0a96f02d4b1b86a4d9520ba1f Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 20 Jun 2022 18:20:28 +0200 Subject: [PATCH 49/54] cluster-network: add strategies for conventionals --- scripts/cluster_network.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 642db4da..1d5608e2 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -281,7 +281,14 @@ def clustering_for_n_clusters(n, n_clusters, custom_busmap=False, aggregate_carr aggregate_generators_carriers=aggregate_carriers, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=line_length_factor, - generator_strategies={'p_nom_max': p_nom_max_strategy, 'p_nom_min': pd.Series.sum}, + generator_strategies={'p_nom_max': p_nom_max_strategy, + 'p_nom_min': pd.Series.sum, + 'p_min_pu': pd.Series.mean, + 'marginal_cost': pd.Series.mean, + 'committable': np.any, + 'ramp_limit_up': pd.Series.max, + 'ramp_limit_down': pd.Series.max, + }, scale_link_capital_costs=False) if not n.links.empty: From 51ff3f02bb5ac45f840b7cd1a15c41a6e4112f6a Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Tue, 21 Jun 2022 16:13:16 +0200 Subject: [PATCH 50/54] helpers: check snakemake version for bug fix --- scripts/_helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 766fb421..af6d831c 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -231,6 +231,7 @@ def mock_snakemake(rulename, **wildcards): import os from pypsa.descriptors import Dict from snakemake.script import Snakemake + from packaging.version import Version, parse script_dir = Path(__file__).parent.resolve() assert Path.cwd().resolve() == script_dir, \ @@ -240,7 +241,8 @@ def mock_snakemake(rulename, **wildcards): if os.path.exists(p): snakefile = p break - workflow = sm.Workflow(snakefile, overwrite_configfiles=[], rerun_triggers=[]) + kwargs=dict(rerun_triggers=[]) if parse(sm.__version__) > Version("7.7.0") else {} + workflow = sm.Workflow(snakefile, overwrite_configfiles=[], **kwargs) workflow.include(snakefile) workflow.global_resources = {} rule = workflow.get_rule(rulename) From cc657b762874994b06809adedaf5647e014ff83a Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 21 Jun 2022 16:21:21 +0200 Subject: [PATCH 51/54] Update scripts/_helpers.py --- scripts/_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index af6d831c..6e47c053 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -241,7 +241,7 @@ def mock_snakemake(rulename, **wildcards): if os.path.exists(p): snakefile = p break - kwargs=dict(rerun_triggers=[]) if parse(sm.__version__) > Version("7.7.0") else {} + kwargs = dict(rerun_triggers=[]) if parse(sm.__version__) > Version("7.7.0") else {} workflow = sm.Workflow(snakefile, overwrite_configfiles=[], **kwargs) workflow.include(snakefile) workflow.global_resources = {} From d18867ce61c7e6d60ad8ef3ba557d8b03bbefd3b Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 23 Jun 2022 21:27:18 +0200 Subject: [PATCH 52/54] build_renewable_profiles: use dask client instead of kwargs --- scripts/build_renewable_profiles.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 70cadab4..b77d79e1 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -189,6 +189,7 @@ import logging from pypsa.geo import haversine from shapely.geometry import LineString import time +from dask.distributed import Client from _helpers import configure_logging @@ -216,6 +217,7 @@ if __name__ == '__main__': if correction_factor != 1.: logger.info(f'correction_factor is set as {correction_factor}') + client = Client(n_workers=nprocesses) cutout = atlite.Cutout(snakemake.input['cutout']) regions = gpd.read_file(snakemake.input.regions).set_index('name').rename_axis('bus') @@ -266,7 +268,7 @@ if __name__ == '__main__': potential = capacity_per_sqkm * availability.sum('bus') * area func = getattr(cutout, resource.pop('method')) - resource['dask_kwargs'] = {'num_workers': nprocesses} + # resource['dask_kwargs'] = {'num_workers': nprocesses, "scheduler": "threading"} capacity_factor = correction_factor * func(capacity_factor=True, **resource) layout = capacity_factor * area * capacity_per_sqkm profile, capacities = func(matrix=availability.stack(spatial=['y','x']), From 743fdea874aac727e356ba6acf9c787a9eed4cc3 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 23 Jun 2022 21:39:28 +0200 Subject: [PATCH 53/54] add dask-worker-space to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b4734ab2..559dde47 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ gurobi.log /data /data/links_p_nom.csv /cutouts +/dask-worker-space doc/_build From 75f9719076c39d7d6ff3653917d16b966c5a3e07 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 24 Jun 2022 14:07:51 +0200 Subject: [PATCH 54/54] build_renewable_profiles: use LocalCluster instance --- scripts/build_renewable_profiles.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index b77d79e1..37e1e9de 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -189,7 +189,7 @@ import logging from pypsa.geo import haversine from shapely.geometry import LineString import time -from dask.distributed import Client +from dask.distributed import Client, LocalCluster from _helpers import configure_logging @@ -217,8 +217,9 @@ if __name__ == '__main__': if correction_factor != 1.: logger.info(f'correction_factor is set as {correction_factor}') - client = Client(n_workers=nprocesses) - + cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) + client = Client(cluster, asynchronous=True) + cutout = atlite.Cutout(snakemake.input['cutout']) regions = gpd.read_file(snakemake.input.regions).set_index('name').rename_axis('bus') buses = regions.index @@ -268,7 +269,7 @@ if __name__ == '__main__': potential = capacity_per_sqkm * availability.sum('bus') * area func = getattr(cutout, resource.pop('method')) - # resource['dask_kwargs'] = {'num_workers': nprocesses, "scheduler": "threading"} + resource['dask_kwargs'] = {"scheduler": client} capacity_factor = correction_factor * func(capacity_factor=True, **resource) layout = capacity_factor * area * capacity_per_sqkm profile, capacities = func(matrix=availability.stack(spatial=['y','x']),