From e9c5af27a31e2bded864035c7f8bfbaba48e0311 Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Sun, 7 May 2023 14:10:18 +0100 Subject: [PATCH 01/21] bump python version and add setuptools, myst-parser --- .readthedocs.yml | 6 +++++- doc/requirements.txt | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 3d7a86b2..c74e12a2 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -4,8 +4,12 @@ version: 2 +build: + os: ubuntu-22.04 + tools: + python: "3.11" + python: - version: 3.8 install: - requirements: doc/requirements.txt system_packages: true diff --git a/doc/requirements.txt b/doc/requirements.txt index d5c71da9..2f08b8d9 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -2,9 +2,11 @@ # # SPDX-License-Identifier: CC0-1.0 +setuptools sphinx sphinx_book_theme sphinxcontrib-bibtex +myst-parser # recommark is deprecated, https://stackoverflow.com/a/71660856/13573820 pypsa vresutils>=0.3.1 From 510e544a4fd9350671eead4df41ae9d39abc9758 Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Sun, 7 May 2023 14:19:37 +0100 Subject: [PATCH 02/21] add release note --- doc/release_notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 9be10a4a..8e6bfdd4 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -17,7 +17,7 @@ Upcoming Release * Renamed script file from PyPSA-EUR ``build_load_data`` to ``build_electricity_demand`` and ``retrieve_load_data`` to ``retrieve_electricity_demand``. - +* Fix docs readthedocs built * Add plain hydrogen turbine as additional re-electrification option besides hydrogen fuel cell. Add switches for both re-electrification options under From 8a42ff77ad9e37f55eba2252e0e942902f5be08c Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Sun, 7 May 2023 14:24:58 +0100 Subject: [PATCH 03/21] add myst_parser to conf.py --- doc/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/conf.py b/doc/conf.py index 4b3978af..8111c86c 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -36,6 +36,7 @@ sys.path.insert(0, os.path.abspath("../scripts")) extensions = [ #'sphinx.ext.autodoc', #'sphinx.ext.autosummary', + "myst_parser", "sphinx.ext.autosectionlabel", "sphinx.ext.intersphinx", "sphinx.ext.todo", From e62d947fb850686440bb2f55df74d994a854e35d Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Sun, 7 May 2023 14:29:27 +0100 Subject: [PATCH 04/21] set system_packages false --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index c74e12a2..35eff5de 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,4 +12,4 @@ build: python: install: - requirements: doc/requirements.txt - system_packages: true + system_packages: false From 5bfa6237e8eaacad41ed9004dda86f4b1651afb4 Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Sun, 7 May 2023 15:44:33 +0100 Subject: [PATCH 05/21] add graphviz --- .readthedocs.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index 35eff5de..b5dc5eba 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -8,8 +8,11 @@ build: os: ubuntu-22.04 tools: python: "3.11" + apt_packages: + - graphviz python: install: - requirements: doc/requirements.txt system_packages: false + apt_packages: From e18b188e81f64656b45232da8ec15c9954ef2281 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 7 May 2023 14:44:54 +0000 Subject: [PATCH 06/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index b5dc5eba..0ee32009 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ build: tools: python: "3.11" apt_packages: - - graphviz + - graphviz python: install: From dad04a64a1db152813c11ec02dd30c4b03747e1d Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Sun, 7 May 2023 15:57:42 +0100 Subject: [PATCH 07/21] Update .readthedocs.yml --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 0ee32009..b5dc5eba 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ build: tools: python: "3.11" apt_packages: - - graphviz + - graphviz python: install: From c431cf6f44a221dc6e7971d4977a1575c6d16867 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 7 May 2023 14:57:56 +0000 Subject: [PATCH 08/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index b5dc5eba..0ee32009 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ build: tools: python: "3.11" apt_packages: - - graphviz + - graphviz python: install: From 702bb59a18ade47321440112ead6598bb0e4a0d2 Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Sun, 7 May 2023 16:06:33 +0100 Subject: [PATCH 09/21] fix artefact --- .readthedocs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 0ee32009..900dba1e 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -15,4 +15,3 @@ python: install: - requirements: doc/requirements.txt system_packages: false - apt_packages: From 8f7667f0f90048bafdff1f35d7ded7993eb68c83 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 22:30:27 +0000 Subject: [PATCH 10/21] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/docformatter: v1.6.4 → v1.6.5](https://github.com/PyCQA/docformatter/compare/v1.6.4...v1.6.5) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 02471b57..88685f3b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: # Make docstrings PEP 257 compliant - repo: https://github.com/PyCQA/docformatter - rev: v1.6.4 + rev: v1.6.5 hooks: - id: docformatter args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"] From 5c2e3c9fc136b12ea8953fada095d335541d22ca Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Wed, 10 May 2023 09:58:25 +0200 Subject: [PATCH 11/21] add Shape2Shapes in add_electricit.py --- scripts/add_electricity.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 1d32bce1..64cf442e 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -94,7 +94,9 @@ import pypsa import xarray as xr from _helpers import configure_logging, update_p_nom_max from powerplantmatching.export import map_country_bus -from vresutils import transfer as vtransfer +from itertools import product +import scipy.sparse as sparse +from shapely.prepared import prep idx = pd.IndexSlice @@ -215,6 +217,20 @@ def load_powerplants(ppl_fn): .replace({"carrier": carrier_dict}) ) +def Shapes2Shapes(orig, dest): + """ + Adopted from vresutils.transfer.Shapes2Shapes() + """ + orig_prepped = list(map(prep, orig)) + transfer = sparse.lil_matrix((len(dest), len(orig)), dtype=float) + + for i,j in product(range(len(dest)), range(len(orig))): + if orig_prepped[j].intersects(dest[i]): + area = orig[j].intersection(dest[i]).area + transfer[i,j] = area/dest[i].area + + return transfer + def attach_load(n, regions, load, nuts3_shapes, countries, scaling=1.0): substation_lv_i = n.buses.index[n.buses["substation_lv"]] @@ -232,7 +248,7 @@ def attach_load(n, regions, load, nuts3_shapes, countries, scaling=1.0): return pd.DataFrame({group.index[0]: l}) else: nuts3_cntry = nuts3.loc[nuts3.country == cntry] - transfer = vtransfer.Shapes2Shapes( + transfer = Shapes2Shapes( group, nuts3_cntry.geometry, normed=False ).T.tocsr() gdp_n = pd.Series( From e91068196b9db670ca95d00184e273f7890be61b Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Wed, 10 May 2023 10:02:41 +0200 Subject: [PATCH 12/21] replace vresutils annuity function with add_electricity.py calculate_annuity --- scripts/prepare_sector_network.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 1cb7146e..944c3caa 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -28,7 +28,7 @@ from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentati from pypsa.geo import haversine_pts from pypsa.io import import_components_from_dataframe from scipy.stats import beta -from vresutils.costdata import annuity +from add_electricity import calculate_annuity logger = logging.getLogger(__name__) @@ -742,7 +742,7 @@ def prepare_costs(cost_file, config, nyears): costs = costs.fillna(config["fill_values"]) def annuity_factor(v): - return annuity(v["lifetime"], v["discount rate"]) + v["FOM"] / 100 + return calculate_annuity(v["lifetime"], v["discount rate"]) + v["FOM"] / 100 costs["fixed"] = [ annuity_factor(v) * v["investment"] * nyears for i, v in costs.iterrows() @@ -851,7 +851,7 @@ def add_wave(n, wave_cost_factor): capacity = pd.Series({"Attenuator": 750, "F2HB": 1000, "MultiPA": 600}) # in EUR/MW - annuity_factor = annuity(25, 0.07) + 0.03 + annuity_factor = calculate_annuity(25, 0.07) + 0.03 costs = ( 1e6 * wave_cost_factor From b21965a98600b4fcd3bbf286ac9b5f9bd9032fa6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 May 2023 08:09:43 +0000 Subject: [PATCH 13/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_electricity.py | 9 +++++---- scripts/prepare_sector_network.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 64cf442e..83ec4bb9 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -85,17 +85,17 @@ It further adds extendable ``generators`` with **zero** capacity for """ import logging +from itertools import product import geopandas as gpd import numpy as np import pandas as pd import powerplantmatching as pm import pypsa +import scipy.sparse as sparse import xarray as xr from _helpers import configure_logging, update_p_nom_max from powerplantmatching.export import map_country_bus -from itertools import product -import scipy.sparse as sparse from shapely.prepared import prep idx = pd.IndexSlice @@ -217,6 +217,7 @@ def load_powerplants(ppl_fn): .replace({"carrier": carrier_dict}) ) + def Shapes2Shapes(orig, dest): """ Adopted from vresutils.transfer.Shapes2Shapes() @@ -224,10 +225,10 @@ def Shapes2Shapes(orig, dest): orig_prepped = list(map(prep, orig)) transfer = sparse.lil_matrix((len(dest), len(orig)), dtype=float) - for i,j in product(range(len(dest)), range(len(orig))): + for i, j in product(range(len(dest)), range(len(orig))): if orig_prepped[j].intersects(dest[i]): area = orig[j].intersection(dest[i]).area - transfer[i,j] = area/dest[i].area + transfer[i, j] = area / dest[i].area return transfer diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 944c3caa..012c9714 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -22,13 +22,13 @@ from _helpers import ( override_component_attrs, update_config_with_sector_opts, ) +from add_electricity import calculate_annuity from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2 from networkx.algorithms import complement from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation from pypsa.geo import haversine_pts from pypsa.io import import_components_from_dataframe from scipy.stats import beta -from add_electricity import calculate_annuity logger = logging.getLogger(__name__) From 654f46f8683cf0613db90d38ab1127a9db62ffd4 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Thu, 11 May 2023 16:58:35 +0200 Subject: [PATCH 14/21] alpha testing complete vresutils removal --- envs/environment.fixed.yaml | 3 +-- envs/environment.yaml | 3 +-- scripts/add_electricity.py | 6 ++---- scripts/solve_network.py | 31 ++++++++++++----------------- scripts/solve_operations_network.py | 30 ++++++++++++---------------- 5 files changed, 30 insertions(+), 43 deletions(-) diff --git a/envs/environment.fixed.yaml b/envs/environment.fixed.yaml index 7d8fcc45..1ff9313d 100644 --- a/envs/environment.fixed.yaml +++ b/envs/environment.fixed.yaml @@ -226,7 +226,7 @@ dependencies: - nspr=4.35 - nss=3.88 - numexpr=2.8.3 -- numpy=1.23.5 +- numpy=1.24 - openjdk=17.0.3 - openjpeg=2.5.0 - openpyxl=3.1.0 @@ -378,4 +378,3 @@ dependencies: - highspy==1.5.0.dev0 - pybind11==2.10.3 - tsam==2.2.2 - - vresutils==0.3.1 diff --git a/envs/environment.yaml b/envs/environment.yaml index 0a9891a5..f970c9ba 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -25,7 +25,7 @@ dependencies: - pytables - lxml - powerplantmatching>=0.5.5 -- numpy<1.24 +- numpy - pandas>=1.4 - geopandas>=0.11.0 - xarray @@ -55,5 +55,4 @@ dependencies: - rasterio!=1.2.10 - pip: - - vresutils>=0.3.1 - tsam>=1.1.0 diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 83ec4bb9..f910dee4 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -218,7 +218,7 @@ def load_powerplants(ppl_fn): ) -def Shapes2Shapes(orig, dest): +def shapes_to_shapes(orig, dest): """ Adopted from vresutils.transfer.Shapes2Shapes() """ @@ -249,9 +249,7 @@ def attach_load(n, regions, load, nuts3_shapes, countries, scaling=1.0): return pd.DataFrame({group.index[0]: l}) else: nuts3_cntry = nuts3.loc[nuts3.country == cntry] - transfer = Shapes2Shapes( - group, nuts3_cntry.geometry, normed=False - ).T.tocsr() + transfer = shapes_to_shapes(group, nuts3_cntry.geometry).T.tocsr() gdp_n = pd.Series( transfer.dot(nuts3_cntry["gdp"].fillna(1.0).values), index=group.index ) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index ff1c0ccf..53ac8816 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -38,7 +38,6 @@ from _helpers import ( override_component_attrs, update_config_with_sector_opts, ) -from vresutils.benchmark import memory_logger logger = logging.getLogger(__name__) pypsa.pf.logger.setLevel(logging.WARNING) @@ -667,23 +666,19 @@ if __name__ == "__main__": np.random.seed(solve_opts.get("seed", 123)) - fn = getattr(snakemake.log, "memory", None) - with memory_logger(filename=fn, interval=30.0) as mem: - if "overrides" in snakemake.input.keys(): - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network( - snakemake.input.network, override_component_attrs=overrides - ) - else: - n = pypsa.Network(snakemake.input.network) - - n = prepare_network(n, solve_opts, config=snakemake.config) - - n = solve_network( - n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver + if "overrides" in snakemake.input.keys(): + overrides = override_component_attrs(snakemake.input.overrides) + n = pypsa.Network( + snakemake.input.network, override_component_attrs=overrides ) + else: + n = pypsa.Network(snakemake.input.network) - n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) - n.export_to_netcdf(snakemake.output[0]) + n = prepare_network(n, solve_opts, config=snakemake.config) - logger.info("Maximum memory usage: {}".format(mem.mem_usage)) + n = solve_network( + n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver + ) + + n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) + n.export_to_netcdf(snakemake.output[0]) diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 25fe0753..9c2fe3c1 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -46,23 +46,19 @@ if __name__ == "__main__": np.random.seed(solve_opts.get("seed", 123)) - fn = getattr(snakemake.log, "memory", None) - with memory_logger(filename=fn, interval=30.0) as mem: - if "overrides" in snakemake.input: - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network( - snakemake.input.network, override_component_attrs=overrides - ) - else: - n = pypsa.Network(snakemake.input.network) - - n.optimize.fix_optimal_capacities() - n = prepare_network(n, solve_opts, config=snakemake.config) - n = solve_network( - n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver + if "overrides" in snakemake.input: + overrides = override_component_attrs(snakemake.input.overrides) + n = pypsa.Network( + snakemake.input.network, override_component_attrs=overrides ) + else: + n = pypsa.Network(snakemake.input.network) - n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) - n.export_to_netcdf(snakemake.output[0]) + n.optimize.fix_optimal_capacities() + n = prepare_network(n, solve_opts, config=snakemake.config) + n = solve_network( + n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver + ) - logger.info("Maximum memory usage: {}".format(mem.mem_usage)) + n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) + n.export_to_netcdf(snakemake.output[0]) From 7d6d6d2805f34a6de6d6e1400d376e8970886c4d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 May 2023 14:59:10 +0000 Subject: [PATCH 15/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/solve_network.py | 4 +--- scripts/solve_operations_network.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 53ac8816..8f4fdf5b 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -668,9 +668,7 @@ if __name__ == "__main__": if "overrides" in snakemake.input.keys(): overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network( - snakemake.input.network, override_component_attrs=overrides - ) + n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) else: n = pypsa.Network(snakemake.input.network) diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 9c2fe3c1..987302bb 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -48,9 +48,7 @@ if __name__ == "__main__": if "overrides" in snakemake.input: overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network( - snakemake.input.network, override_component_attrs=overrides - ) + n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) else: n = pypsa.Network(snakemake.input.network) From e29571535b504c2db3733e877ca0d0c70248b07f Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 12 May 2023 07:11:41 +0200 Subject: [PATCH 16/21] remove references to memory logfile in *.smk, add release note --- doc/release_notes.rst | 2 ++ doc/requirements.txt | 1 - rules/solve_electricity.smk | 4 ---- rules/solve_myopic.smk | 2 -- rules/solve_overnight.smk | 2 -- scripts/solve_operations_network.py | 1 - 6 files changed, 2 insertions(+), 10 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 8e6bfdd4..14f2939d 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -23,6 +23,8 @@ Upcoming Release hydrogen fuel cell. Add switches for both re-electrification options under ``sector: hydrogen_turbine:`` and ``sector: hydrogen_fuel_cell:``. +* Remove ``vresutils`` dependency. + PyPSA-Eur 0.8.0 (18th March 2023) ================================= diff --git a/doc/requirements.txt b/doc/requirements.txt index 2f08b8d9..3e760c81 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -9,7 +9,6 @@ sphinxcontrib-bibtex myst-parser # recommark is deprecated, https://stackoverflow.com/a/71660856/13573820 pypsa -vresutils>=0.3.1 powerplantmatching>=0.5.5 atlite>=0.2.9 dask[distributed] diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 8ddeca92..fc70ce42 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -14,8 +14,6 @@ rule solve_network: ), python=LOGS + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", - memory=LOGS - + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_memory.log", benchmark: BENCHMARKS + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 4 @@ -41,8 +39,6 @@ rule solve_operations_network: ), python=LOGS + "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", - memory=LOGS - + "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_memory.log", benchmark: ( BENCHMARKS diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index f10d8157..ec4281ff 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -90,8 +90,6 @@ rule solve_sector_network_myopic: + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log", python=LOGS + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log", - memory=LOGS - + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_memory.log", threads: 4 resources: mem_mb=config["solving"]["mem"], diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index c2e103e5..f05925b0 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -21,8 +21,6 @@ rule solve_sector_network: + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log", python=LOGS + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log", - memory=LOGS - + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_memory.log", threads: config["solving"]["solver"].get("threads", 4) resources: mem_mb=config["solving"]["mem"], diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 987302bb..27520485 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -17,7 +17,6 @@ from _helpers import ( update_config_with_sector_opts, ) from solve_network import prepare_network, solve_network -from vresutils.benchmark import memory_logger logger = logging.getLogger(__name__) From 5be21dfc5a4cefd5e01bbf7798663d18eeacc638 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 12 May 2023 09:07:16 +0200 Subject: [PATCH 17/21] add option for piecewise linear transmission loss approximation --- config/config.default.yaml | 2 +- doc/configtables/solving.csv | 2 +- doc/release_notes.rst | 4 ++++ envs/environment.yaml | 2 +- scripts/solve_network.py | 3 +++ 5 files changed, 10 insertions(+), 3 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 10e6a6ed..78c1385c 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -623,9 +623,9 @@ clustering: solving: #tmpdir: "path/to/tmp" options: - formulation: kirchhoff clip_p_max_pu: 1.e-2 load_shedding: false + transmission_losses: 0 noisy_costs: true skip_iterations: true track_iterations: false diff --git a/doc/configtables/solving.csv b/doc/configtables/solving.csv index cba28cbe..c252ff32 100644 --- a/doc/configtables/solving.csv +++ b/doc/configtables/solving.csv @@ -1,7 +1,7 @@ ,Unit,Values,Description options,,, --- formulation,--,"Any of {'angles', 'kirchhoff', 'cycles', 'ptdf'}","Specifies which variant of linearized power flow formulations to use in the optimisation problem. Recommended is 'kirchhoff'. Explained in `this article `_." -- load_shedding,bool/float,"{'true','false', float}","Add generators with very high marginal cost to simulate load shedding and avoid problem infeasibilities. If load shedding is a float, it denotes the marginal cost in EUR/kWh." +-- transmission_losses,int,"[0-9]","Add piecewise linear approximation of transmission losses based on n tangents. Defaults to 0, which means losses are ignored." -- noisy_costs,bool,"{'true','false'}","Add random noise to marginal cost of generators by :math:`\mathcal{U}(0.009,0,011)` and capital cost of lines and links by :math:`\mathcal{U}(0.09,0,11)`." -- min_iterations,--,int,"Minimum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run." -- max_iterations,--,int,"Maximum number of solving iterations in between which resistance and reactence (``x/r``) are updated for branches according to ``s_nom_opt`` of the previous run." diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 14f2939d..3af16477 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -25,6 +25,10 @@ Upcoming Release * Remove ``vresutils`` dependency. +* Add option to include a piecewise linear approximation of transmission losses, + e.g. by setting ``solving: options: transmission_losses: 2`` for an + approximation with two tangents. + PyPSA-Eur 0.8.0 (18th March 2023) ================================= diff --git a/envs/environment.yaml b/envs/environment.yaml index f970c9ba..9d800fdc 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -10,7 +10,7 @@ dependencies: - python>=3.8 - pip -- pypsa>=0.21.3 +- pypsa>=0.23 - atlite>=0.2.9 - dask diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 8f4fdf5b..e671ffd3 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -600,6 +600,7 @@ def solve_network(n, config, opts="", **kwargs): track_iterations = cf_solving.get("track_iterations", False) min_iterations = cf_solving.get("min_iterations", 4) max_iterations = cf_solving.get("max_iterations", 6) + transmission_losses = cf_solving.get("transmission_losses", 0) # add to network for extra_functionality n.config = config @@ -613,6 +614,7 @@ def solve_network(n, config, opts="", **kwargs): if skip_iterations: status, condition = n.optimize( solver_name=solver_name, + transmission_losses=transmission_losses, extra_functionality=extra_functionality, **solver_options, **kwargs, @@ -623,6 +625,7 @@ def solve_network(n, config, opts="", **kwargs): track_iterations=track_iterations, min_iterations=min_iterations, max_iterations=max_iterations, + transmission_losses=transmission_losses, extra_functionality=extra_functionality, **solver_options, **kwargs, From 67ef38b35eb2e46d8dcc5252216b73ee062465f0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 12 May 2023 13:59:41 +0200 Subject: [PATCH 18/21] retrieve*: handle extraction path via snakemake.output --- scripts/retrieve_databundle.py | 3 +-- scripts/retrieve_gas_infrastructure_data.py | 2 +- scripts/retrieve_sector_databundle.py | 16 +++++++++------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index 0c6a7feb..75d8519e 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -58,9 +58,8 @@ if __name__ == "__main__": else: url = "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz" - # Save locations tarball_fn = Path(f"{rootpath}/bundle.tar.xz") - to_fn = Path(f"{rootpath}/data") + to_fn = Path(rootpath) / Path(snakemake.output[0]).parent.parent logger.info(f"Downloading databundle from '{url}'.") disable_progress = snakemake.config["run"].get("disable_progressbar", False) diff --git a/scripts/retrieve_gas_infrastructure_data.py b/scripts/retrieve_gas_infrastructure_data.py index dda7bd8c..42b726db 100644 --- a/scripts/retrieve_gas_infrastructure_data.py +++ b/scripts/retrieve_gas_infrastructure_data.py @@ -29,7 +29,7 @@ if __name__ == "__main__": # Save locations zip_fn = Path(f"{rootpath}/IGGIELGN.zip") - to_fn = Path(f"{rootpath}/data/gas_network/scigrid-gas") + to_fn = Path(rootpath) / Path(snakemake.output[0]).parent.parent logger.info(f"Downloading databundle from '{url}'.") disable_progress = snakemake.config["run"].get("disable_progressbar", False) diff --git a/scripts/retrieve_sector_databundle.py b/scripts/retrieve_sector_databundle.py index 97426ab2..0991bbe3 100644 --- a/scripts/retrieve_sector_databundle.py +++ b/scripts/retrieve_sector_databundle.py @@ -10,23 +10,25 @@ import logging logger = logging.getLogger(__name__) -import os -import sys import tarfile from pathlib import Path -# Add pypsa-eur scripts to path for import of _helpers -sys.path.insert(0, os.getcwd() + "/../pypsa-eur/scripts") - from _helpers import configure_logging, progress_retrieve if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake("retrieve_databundle") + rootpath = ".." + else: + rootpath = "." configure_logging(snakemake) url = "https://zenodo.org/record/5824485/files/pypsa-eur-sec-data-bundle.tar.gz" - tarball_fn = Path("sector-bundle.tar.gz") - to_fn = Path("data") + tarball_fn = Path(f"{rootpath}/sector-bundle.tar.gz") + to_fn = Path(rootpath) / Path(snakemake.output[0]).parent logger.info(f"Downloading databundle from '{url}'.") disable_progress = snakemake.config["run"].get("disable_progressbar", False) From b2216355f10bf11147cc7f03e74b17f6dc4b60e6 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 12 May 2023 14:10:04 +0200 Subject: [PATCH 19/21] retrieve_sector_data: add another .parent --- scripts/retrieve_sector_databundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/retrieve_sector_databundle.py b/scripts/retrieve_sector_databundle.py index 0991bbe3..0d172c8d 100644 --- a/scripts/retrieve_sector_databundle.py +++ b/scripts/retrieve_sector_databundle.py @@ -28,7 +28,7 @@ if __name__ == "__main__": url = "https://zenodo.org/record/5824485/files/pypsa-eur-sec-data-bundle.tar.gz" tarball_fn = Path(f"{rootpath}/sector-bundle.tar.gz") - to_fn = Path(rootpath) / Path(snakemake.output[0]).parent + to_fn = Path(rootpath) / Path(snakemake.output[0]).parent.parent logger.info(f"Downloading databundle from '{url}'.") disable_progress = snakemake.config["run"].get("disable_progressbar", False) From 8f91963e7623d6cf574c0fd267b80b581f9560d8 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 12 May 2023 14:50:58 +0200 Subject: [PATCH 20/21] properly reference p_max_pu files --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index f910dee4..85ab35e2 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -418,7 +418,7 @@ def attach_conventional_generators( if f"conventional_{carrier}_{attr}" in conventional_inputs: # Values affecting generators of technology k country-specific # First map generator buses to countries; then map countries to p_max_pu - values = pd.read_csv(values, index_col=0).iloc[:, 0] + values = pd.read_csv(snakemake.input[f"conventional_{carrier}_{attr}"], index_col=0).iloc[:, 0] bus_values = n.buses.country.map(values) n.generators[attr].update( n.generators.loc[idx].bus.map(bus_values).dropna() From 38bae672da42db241eb885eceeb3a6097644eeb5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 May 2023 12:51:39 +0000 Subject: [PATCH 21/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_electricity.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 85ab35e2..4afa3e22 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -418,7 +418,9 @@ def attach_conventional_generators( if f"conventional_{carrier}_{attr}" in conventional_inputs: # Values affecting generators of technology k country-specific # First map generator buses to countries; then map countries to p_max_pu - values = pd.read_csv(snakemake.input[f"conventional_{carrier}_{attr}"], index_col=0).iloc[:, 0] + values = pd.read_csv( + snakemake.input[f"conventional_{carrier}_{attr}"], index_col=0 + ).iloc[:, 0] bus_values = n.buses.country.map(values) n.generators[attr].update( n.generators.loc[idx].bus.map(bus_values).dropna()