From 81e7c4eb67a8385eee5f5701c2ee134b67fd585a Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 29 Jan 2024 12:08:56 +0100 Subject: [PATCH 1/7] remove pyomo dependency in cluster network, use scip as OS solver --- envs/environment.yaml | 3 +-- scripts/cluster_network.py | 41 ++++++++++++++------------------------ 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 6ff4b7f1..895b271a 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -36,7 +36,7 @@ dependencies: - networkx - scipy - shapely>=2.0 -- pyomo +- scipopt - matplotlib - proj - fiona @@ -47,7 +47,6 @@ dependencies: - tabula-py - pyxlsb - graphviz -- ipopt # Keep in conda environment when calling ipython - ipython diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 01af29aa..a18121e8 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -126,10 +126,10 @@ import warnings from functools import reduce import geopandas as gpd +import linopy import matplotlib.pyplot as plt import numpy as np import pandas as pd -import pyomo.environ as po import pypsa import seaborn as sns from _helpers import configure_logging, update_p_nom_max @@ -214,7 +214,7 @@ def get_feature_for_hac(n, buses_i=None, feature=None): return feature_data -def distribute_clusters(n, n_clusters, focus_weights=None, solver_name="cbc"): +def distribute_clusters(n, n_clusters, focus_weights=None, solver_name="scip"): """ Determine the number of clusters per country. """ @@ -254,31 +254,20 @@ def distribute_clusters(n, n_clusters, focus_weights=None, solver_name="cbc"): L.sum(), 1.0, rtol=1e-3 ), f"Country weights L must sum up to 1.0 when distributing clusters. Is {L.sum()}." - m = po.ConcreteModel() - - def n_bounds(model, *n_id): - return (1, N[n_id]) - - m.n = po.Var(list(L.index), bounds=n_bounds, domain=po.Integers) - m.tot = po.Constraint(expr=(po.summation(m.n) == n_clusters)) - m.objective = po.Objective( - expr=sum((m.n[i] - L.loc[i] * n_clusters) ** 2 for i in L.index), - sense=po.minimize, + m = linopy.Model() + clusters = m.add_variables( + lower=1, upper=N, coords=[L.index], name="n", integer=True ) - - opt = po.SolverFactory(solver_name) - if solver_name == "appsi_highs" or not opt.has_capability("quadratic_objective"): - logger.warning( - f"The configured solver `{solver_name}` does not support quadratic objectives. Falling back to `ipopt`." - ) - opt = po.SolverFactory("ipopt") - - results = opt.solve(m) - assert ( - results["Solver"][0]["Status"] == "ok" - ), f"Solver returned non-optimally: {results}" - - return pd.Series(m.n.get_values(), index=L.index).round().astype(int) + m.add_constraints(clusters.sum() == n_clusters, name="tot") + m.objective = ( + clusters * clusters - 2 * clusters * L * n_clusters + ) # + (L * n_clusters) ** 2 (constant) + if solver_name == "gurobi": + logging.getLogger("gurobipy").propagate = False + else: + solver_name = "scip" + m.solve(solver_name=solver_name) + return m.solution["n"].to_series().astype(int) def busmap_for_n_clusters( From ca1628a585dcfe1749c464cdbc5e1f5af7a0405a Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 29 Jan 2024 12:22:42 +0100 Subject: [PATCH 2/7] cluster_network: tidy up --- scripts/cluster_network.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index a18121e8..b63e7f89 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -122,6 +122,7 @@ Exemplary unsolved network clustered to 37 nodes: """ import logging +import os import warnings from functools import reduce @@ -259,12 +260,14 @@ def distribute_clusters(n, n_clusters, focus_weights=None, solver_name="scip"): lower=1, upper=N, coords=[L.index], name="n", integer=True ) m.add_constraints(clusters.sum() == n_clusters, name="tot") - m.objective = ( - clusters * clusters - 2 * clusters * L * n_clusters - ) # + (L * n_clusters) ** 2 (constant) + # leave out constant in objective L * n_clusters ** 2 + m.objective = (clusters * clusters - 2 * clusters * L * n_clusters).sum() if solver_name == "gurobi": logging.getLogger("gurobipy").propagate = False - else: + elif solver_name != "scip": + logger.info( + f"The configured solver `{solver_name}` does not support quadratic objectives. Falling back to `scip`." + ) solver_name = "scip" m.solve(solver_name=solver_name) return m.solution["n"].to_series().astype(int) @@ -374,7 +377,7 @@ def clustering_for_n_clusters( aggregate_carriers=None, line_length_factor=1.25, aggregation_strategies=dict(), - solver_name="cbc", + solver_name="scip", algorithm="hac", feature=None, extended_link_costs=0, @@ -451,7 +454,6 @@ if __name__ == "__main__": params = snakemake.params solver_name = snakemake.config["solving"]["solver"]["name"] - solver_name = "appsi_highs" if solver_name == "highs" else solver_name n = pypsa.Network(snakemake.input.network) From 56b22a3b4ea3e95e521bf4234b21899a963eefb5 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 30 Jan 2024 10:29:08 +0100 Subject: [PATCH 3/7] env: correct pyscipopt dependency cluster_network: address deprecation warning --- envs/environment.yaml | 2 +- scripts/cluster_network.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 895b271a..d92eb696 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -36,7 +36,7 @@ dependencies: - networkx - scipy - shapely>=2.0 -- scipopt +- pyscipopt - matplotlib - proj - fiona diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index b63e7f89..2c69b9af 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -364,7 +364,7 @@ def busmap_for_n_clusters( return ( n.buses.groupby(["country", "sub_network"], group_keys=False) - .apply(busmap_for_country) + .apply(busmap_for_country, include_groups=False) .squeeze() .rename("busmap") ) From 662252d20fa4ad488c95847c8ea1136ed6d0b629 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 30 Jan 2024 11:06:05 +0100 Subject: [PATCH 4/7] update release notes --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 86763287..c20f1ebe 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -60,6 +60,8 @@ Upcoming Release * The rule ``plot_network`` has been split into separate rules for plotting electricity, hydrogen and gas networks. +* To determine the optimal topology to meet the number of clusters, the workflow used pyomo in combination with ``ipopt`` or ``gurobi``. This dependency has been replaced by using ``linopy`` in combination with ``scipopt`` or ``gurobi``. The environment file has been updated accordingly. + PyPSA-Eur 0.9.0 (5th January 2024) ================================== From 9c558a3e460355d212d8d4e28f7e4f0dd87441ae Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Tue, 30 Jan 2024 17:37:44 +0100 Subject: [PATCH 5/7] Update scripts/cluster_network.py Co-authored-by: Fabian Neumann --- scripts/cluster_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 2c69b9af..39b6ad96 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -260,7 +260,7 @@ def distribute_clusters(n, n_clusters, focus_weights=None, solver_name="scip"): lower=1, upper=N, coords=[L.index], name="n", integer=True ) m.add_constraints(clusters.sum() == n_clusters, name="tot") - # leave out constant in objective L * n_clusters ** 2 + # leave out constant in objective (L * n_clusters) ** 2 m.objective = (clusters * clusters - 2 * clusters * L * n_clusters).sum() if solver_name == "gurobi": logging.getLogger("gurobipy").propagate = False From b7750d21eaa58fd1f03af8b51ce39dd745290f71 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 31 Jan 2024 09:44:13 +0100 Subject: [PATCH 6/7] CI: use scip and highs as solvers for clustering and solving, instead of ipopt and glpk --- .github/workflows/ci.yaml | 10 ---------- config/test/config.electricity.yaml | 4 ++-- config/test/config.myopic.yaml | 4 ++-- config/test/config.overnight.yaml | 4 ++-- config/test/config.perfect.yaml | 4 ++-- doc/release_notes.rst | 5 +++++ envs/environment.yaml | 1 + 7 files changed, 14 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c0fb745d..cf52172f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -46,16 +46,6 @@ jobs: run: | echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc - - name: Add solver to environment - run: | - echo -e "- glpk\n- ipopt<3.13.3" >> envs/environment.yaml - if: ${{ matrix.os }} == 'windows-latest' - - - name: Add solver to environment - run: | - echo -e "- glpk\n- ipopt" >> envs/environment.yaml - if: ${{ matrix.os }} != 'windows-latest' - - name: Setup micromamba uses: mamba-org/setup-micromamba@v1 with: diff --git a/config/test/config.electricity.yaml b/config/test/config.electricity.yaml index b750bf62..ef9bfdc0 100644 --- a/config/test/config.electricity.yaml +++ b/config/test/config.electricity.yaml @@ -69,8 +69,8 @@ lines: solving: solver: - name: glpk - options: "glpk-default" + name: highs + options: "highs-default" plotting: diff --git a/config/test/config.myopic.yaml b/config/test/config.myopic.yaml index 2dab7b04..1da51b36 100644 --- a/config/test/config.myopic.yaml +++ b/config/test/config.myopic.yaml @@ -73,8 +73,8 @@ industry: solving: solver: - name: glpk - options: glpk-default + name: highs + options: highs-default mem: 4000 plotting: diff --git a/config/test/config.overnight.yaml b/config/test/config.overnight.yaml index 6d1900cf..dffdab5a 100644 --- a/config/test/config.overnight.yaml +++ b/config/test/config.overnight.yaml @@ -65,8 +65,8 @@ sector: solving: solver: - name: glpk - options: glpk-default + name: highs + options: highs-default mem: 4000 plotting: diff --git a/config/test/config.perfect.yaml b/config/test/config.perfect.yaml index f20a2c9f..8a5a9cce 100644 --- a/config/test/config.perfect.yaml +++ b/config/test/config.perfect.yaml @@ -75,8 +75,8 @@ industry: solving: solver: - name: glpk - options: glpk-default + name: highs + options: highs-default mem: 4000 plotting: diff --git a/doc/release_notes.rst b/doc/release_notes.rst index c20f1ebe..55e0ad97 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -62,6 +62,11 @@ Upcoming Release * To determine the optimal topology to meet the number of clusters, the workflow used pyomo in combination with ``ipopt`` or ``gurobi``. This dependency has been replaced by using ``linopy`` in combination with ``scipopt`` or ``gurobi``. The environment file has been updated accordingly. +* The ``highs`` solver was added to the default environment file. + +* The default solver for testing the workflow in the CI has been changed from ``glpk`` to ``highs``. + + PyPSA-Eur 0.9.0 (5th January 2024) ================================== diff --git a/envs/environment.yaml b/envs/environment.yaml index d92eb696..235d7729 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -59,3 +59,4 @@ dependencies: - pip: - tsam>=2.3.1 + - highspy From bd55e368f7510aacfc234def4af4b525c4b740ab Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 31 Jan 2024 13:04:10 +0100 Subject: [PATCH 7/7] test: revert setting highs as default solver, as not available for macos --- config/test/config.electricity.yaml | 4 ++-- config/test/config.myopic.yaml | 4 ++-- config/test/config.overnight.yaml | 4 ++-- config/test/config.perfect.yaml | 4 ++-- doc/release_notes.rst | 2 -- envs/environment.yaml | 1 + 6 files changed, 9 insertions(+), 10 deletions(-) diff --git a/config/test/config.electricity.yaml b/config/test/config.electricity.yaml index ef9bfdc0..b750bf62 100644 --- a/config/test/config.electricity.yaml +++ b/config/test/config.electricity.yaml @@ -69,8 +69,8 @@ lines: solving: solver: - name: highs - options: "highs-default" + name: glpk + options: "glpk-default" plotting: diff --git a/config/test/config.myopic.yaml b/config/test/config.myopic.yaml index 1da51b36..2dab7b04 100644 --- a/config/test/config.myopic.yaml +++ b/config/test/config.myopic.yaml @@ -73,8 +73,8 @@ industry: solving: solver: - name: highs - options: highs-default + name: glpk + options: glpk-default mem: 4000 plotting: diff --git a/config/test/config.overnight.yaml b/config/test/config.overnight.yaml index dffdab5a..6d1900cf 100644 --- a/config/test/config.overnight.yaml +++ b/config/test/config.overnight.yaml @@ -65,8 +65,8 @@ sector: solving: solver: - name: highs - options: highs-default + name: glpk + options: glpk-default mem: 4000 plotting: diff --git a/config/test/config.perfect.yaml b/config/test/config.perfect.yaml index 8a5a9cce..f20a2c9f 100644 --- a/config/test/config.perfect.yaml +++ b/config/test/config.perfect.yaml @@ -75,8 +75,8 @@ industry: solving: solver: - name: highs - options: highs-default + name: glpk + options: glpk-default mem: 4000 plotting: diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 55e0ad97..6fa88738 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -64,8 +64,6 @@ Upcoming Release * The ``highs`` solver was added to the default environment file. -* The default solver for testing the workflow in the CI has been changed from ``glpk`` to ``highs``. - PyPSA-Eur 0.9.0 (5th January 2024) diff --git a/envs/environment.yaml b/envs/environment.yaml index 235d7729..26e18f0d 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -35,6 +35,7 @@ dependencies: - netcdf4 - networkx - scipy +- glpk - shapely>=2.0 - pyscipopt - matplotlib