From 08a4d196af265ba1e4fc5729b33f654cdd8ffc7b Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 26 Jan 2024 23:16:33 +0100 Subject: [PATCH 01/15] ci: add workflow with inhouse package at current master --- .github/workflows/ci-sync.yaml | 97 ++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 .github/workflows/ci-sync.yaml diff --git a/.github/workflows/ci-sync.yaml b/.github/workflows/ci-sync.yaml new file mode 100644 index 00000000..e5b4006c --- /dev/null +++ b/.github/workflows/ci-sync.yaml @@ -0,0 +1,97 @@ +# SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC0-1.0 + +name: CI - Synced Inhouse Packages + +# Caching method based on and described by: +# epassaro (2021): https://dev.to/epassaro/caching-anaconda-environments-in-github-actions-5hde +# and code in GitHub repo: https://github.com/epassaro/cache-conda-envs + +on: + push: + branches: + - master + pull_request: + branches: + - master + schedule: + - cron: "0 5 * * TUE" + +env: + DATA_CACHE_NUMBER: 2 + +jobs: + build: + + strategy: + fail-fast: false + max-parallel: 1 + + runs-on: ubuntu-latest + + defaults: + run: + shell: bash -l {0} + + steps: + - uses: actions/checkout@v3 + + - name: Setup secrets + run: | + echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc + + - name: Add solver to environment + run: | + echo -e "- glpk\n- ipopt<3.13.3" >> envs/environment.yaml + if: ${{ matrix.os }} == 'windows-latest' + + - name: Add solver to environment + run: | + echo -e "- glpk\n- ipopt" >> envs/environment.yaml + if: ${{ matrix.os }} != 'windows-latest' + + - name: Setup micromamba + uses: mamba-org/setup-micromamba@v1 + with: + micromamba-version: latest + environment-file: envs/environment.yaml + log-level: debug + init-shell: bash + cache-environment: true + cache-downloads: true + + # install inhouse packages from master branch with pip + - name: Install inhouse packages + run: | + pip install git+https://github.com/PyPSA/atlite.git@master git+https://github.com/PyPSA/powerplantmatching.git@master git+https://github.com/PyPSA/linopy.git@master + + + - name: Set cache dates + run: | + echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV + + - name: Cache data and cutouts folders + uses: actions/cache@v3 + with: + path: | + data + cutouts + key: data-cutouts-${{ env.WEEK }}-${{ env.DATA_CACHE_NUMBER }} + + - name: Test snakemake workflow + run: | + snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime + snakemake -call all --configfile config/test/config.overnight.yaml --rerun-triggers=mtime + snakemake -call all --configfile config/test/config.myopic.yaml --rerun-triggers=mtime + snakemake -call all --configfile config/test/config.perfect.yaml --rerun-triggers=mtime + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: resources-results + path: | + resources + results + if-no-files-found: warn + retention-days: 1 From 81e7c4eb67a8385eee5f5701c2ee134b67fd585a Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 29 Jan 2024 12:08:56 +0100 Subject: [PATCH 02/15] remove pyomo dependency in cluster network, use scip as OS solver --- envs/environment.yaml | 3 +-- scripts/cluster_network.py | 41 ++++++++++++++------------------------ 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 6ff4b7f1..895b271a 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -36,7 +36,7 @@ dependencies: - networkx - scipy - shapely>=2.0 -- pyomo +- scipopt - matplotlib - proj - fiona @@ -47,7 +47,6 @@ dependencies: - tabula-py - pyxlsb - graphviz -- ipopt # Keep in conda environment when calling ipython - ipython diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 01af29aa..a18121e8 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -126,10 +126,10 @@ import warnings from functools import reduce import geopandas as gpd +import linopy import matplotlib.pyplot as plt import numpy as np import pandas as pd -import pyomo.environ as po import pypsa import seaborn as sns from _helpers import configure_logging, update_p_nom_max @@ -214,7 +214,7 @@ def get_feature_for_hac(n, buses_i=None, feature=None): return feature_data -def distribute_clusters(n, n_clusters, focus_weights=None, solver_name="cbc"): +def distribute_clusters(n, n_clusters, focus_weights=None, solver_name="scip"): """ Determine the number of clusters per country. """ @@ -254,31 +254,20 @@ def distribute_clusters(n, n_clusters, focus_weights=None, solver_name="cbc"): L.sum(), 1.0, rtol=1e-3 ), f"Country weights L must sum up to 1.0 when distributing clusters. Is {L.sum()}." - m = po.ConcreteModel() - - def n_bounds(model, *n_id): - return (1, N[n_id]) - - m.n = po.Var(list(L.index), bounds=n_bounds, domain=po.Integers) - m.tot = po.Constraint(expr=(po.summation(m.n) == n_clusters)) - m.objective = po.Objective( - expr=sum((m.n[i] - L.loc[i] * n_clusters) ** 2 for i in L.index), - sense=po.minimize, + m = linopy.Model() + clusters = m.add_variables( + lower=1, upper=N, coords=[L.index], name="n", integer=True ) - - opt = po.SolverFactory(solver_name) - if solver_name == "appsi_highs" or not opt.has_capability("quadratic_objective"): - logger.warning( - f"The configured solver `{solver_name}` does not support quadratic objectives. Falling back to `ipopt`." - ) - opt = po.SolverFactory("ipopt") - - results = opt.solve(m) - assert ( - results["Solver"][0]["Status"] == "ok" - ), f"Solver returned non-optimally: {results}" - - return pd.Series(m.n.get_values(), index=L.index).round().astype(int) + m.add_constraints(clusters.sum() == n_clusters, name="tot") + m.objective = ( + clusters * clusters - 2 * clusters * L * n_clusters + ) # + (L * n_clusters) ** 2 (constant) + if solver_name == "gurobi": + logging.getLogger("gurobipy").propagate = False + else: + solver_name = "scip" + m.solve(solver_name=solver_name) + return m.solution["n"].to_series().astype(int) def busmap_for_n_clusters( From ca1628a585dcfe1749c464cdbc5e1f5af7a0405a Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 29 Jan 2024 12:22:42 +0100 Subject: [PATCH 03/15] cluster_network: tidy up --- scripts/cluster_network.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index a18121e8..b63e7f89 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -122,6 +122,7 @@ Exemplary unsolved network clustered to 37 nodes: """ import logging +import os import warnings from functools import reduce @@ -259,12 +260,14 @@ def distribute_clusters(n, n_clusters, focus_weights=None, solver_name="scip"): lower=1, upper=N, coords=[L.index], name="n", integer=True ) m.add_constraints(clusters.sum() == n_clusters, name="tot") - m.objective = ( - clusters * clusters - 2 * clusters * L * n_clusters - ) # + (L * n_clusters) ** 2 (constant) + # leave out constant in objective L * n_clusters ** 2 + m.objective = (clusters * clusters - 2 * clusters * L * n_clusters).sum() if solver_name == "gurobi": logging.getLogger("gurobipy").propagate = False - else: + elif solver_name != "scip": + logger.info( + f"The configured solver `{solver_name}` does not support quadratic objectives. Falling back to `scip`." + ) solver_name = "scip" m.solve(solver_name=solver_name) return m.solution["n"].to_series().astype(int) @@ -374,7 +377,7 @@ def clustering_for_n_clusters( aggregate_carriers=None, line_length_factor=1.25, aggregation_strategies=dict(), - solver_name="cbc", + solver_name="scip", algorithm="hac", feature=None, extended_link_costs=0, @@ -451,7 +454,6 @@ if __name__ == "__main__": params = snakemake.params solver_name = snakemake.config["solving"]["solver"]["name"] - solver_name = "appsi_highs" if solver_name == "highs" else solver_name n = pypsa.Network(snakemake.input.network) From 56b22a3b4ea3e95e521bf4234b21899a963eefb5 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 30 Jan 2024 10:29:08 +0100 Subject: [PATCH 04/15] env: correct pyscipopt dependency cluster_network: address deprecation warning --- envs/environment.yaml | 2 +- scripts/cluster_network.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 895b271a..d92eb696 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -36,7 +36,7 @@ dependencies: - networkx - scipy - shapely>=2.0 -- scipopt +- pyscipopt - matplotlib - proj - fiona diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index b63e7f89..2c69b9af 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -364,7 +364,7 @@ def busmap_for_n_clusters( return ( n.buses.groupby(["country", "sub_network"], group_keys=False) - .apply(busmap_for_country) + .apply(busmap_for_country, include_groups=False) .squeeze() .rename("busmap") ) From 662252d20fa4ad488c95847c8ea1136ed6d0b629 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 30 Jan 2024 11:06:05 +0100 Subject: [PATCH 05/15] update release notes --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 86763287..c20f1ebe 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -60,6 +60,8 @@ Upcoming Release * The rule ``plot_network`` has been split into separate rules for plotting electricity, hydrogen and gas networks. +* To determine the optimal topology to meet the number of clusters, the workflow used pyomo in combination with ``ipopt`` or ``gurobi``. This dependency has been replaced by using ``linopy`` in combination with ``scipopt`` or ``gurobi``. The environment file has been updated accordingly. + PyPSA-Eur 0.9.0 (5th January 2024) ================================== From 9c558a3e460355d212d8d4e28f7e4f0dd87441ae Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Tue, 30 Jan 2024 17:37:44 +0100 Subject: [PATCH 06/15] Update scripts/cluster_network.py Co-authored-by: Fabian Neumann --- scripts/cluster_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 2c69b9af..39b6ad96 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -260,7 +260,7 @@ def distribute_clusters(n, n_clusters, focus_weights=None, solver_name="scip"): lower=1, upper=N, coords=[L.index], name="n", integer=True ) m.add_constraints(clusters.sum() == n_clusters, name="tot") - # leave out constant in objective L * n_clusters ** 2 + # leave out constant in objective (L * n_clusters) ** 2 m.objective = (clusters * clusters - 2 * clusters * L * n_clusters).sum() if solver_name == "gurobi": logging.getLogger("gurobipy").propagate = False From b7750d21eaa58fd1f03af8b51ce39dd745290f71 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 31 Jan 2024 09:44:13 +0100 Subject: [PATCH 07/15] CI: use scip and highs as solvers for clustering and solving, instead of ipopt and glpk --- .github/workflows/ci.yaml | 10 ---------- config/test/config.electricity.yaml | 4 ++-- config/test/config.myopic.yaml | 4 ++-- config/test/config.overnight.yaml | 4 ++-- config/test/config.perfect.yaml | 4 ++-- doc/release_notes.rst | 5 +++++ envs/environment.yaml | 1 + 7 files changed, 14 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c0fb745d..cf52172f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -46,16 +46,6 @@ jobs: run: | echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc - - name: Add solver to environment - run: | - echo -e "- glpk\n- ipopt<3.13.3" >> envs/environment.yaml - if: ${{ matrix.os }} == 'windows-latest' - - - name: Add solver to environment - run: | - echo -e "- glpk\n- ipopt" >> envs/environment.yaml - if: ${{ matrix.os }} != 'windows-latest' - - name: Setup micromamba uses: mamba-org/setup-micromamba@v1 with: diff --git a/config/test/config.electricity.yaml b/config/test/config.electricity.yaml index b750bf62..ef9bfdc0 100644 --- a/config/test/config.electricity.yaml +++ b/config/test/config.electricity.yaml @@ -69,8 +69,8 @@ lines: solving: solver: - name: glpk - options: "glpk-default" + name: highs + options: "highs-default" plotting: diff --git a/config/test/config.myopic.yaml b/config/test/config.myopic.yaml index 2dab7b04..1da51b36 100644 --- a/config/test/config.myopic.yaml +++ b/config/test/config.myopic.yaml @@ -73,8 +73,8 @@ industry: solving: solver: - name: glpk - options: glpk-default + name: highs + options: highs-default mem: 4000 plotting: diff --git a/config/test/config.overnight.yaml b/config/test/config.overnight.yaml index 6d1900cf..dffdab5a 100644 --- a/config/test/config.overnight.yaml +++ b/config/test/config.overnight.yaml @@ -65,8 +65,8 @@ sector: solving: solver: - name: glpk - options: glpk-default + name: highs + options: highs-default mem: 4000 plotting: diff --git a/config/test/config.perfect.yaml b/config/test/config.perfect.yaml index f20a2c9f..8a5a9cce 100644 --- a/config/test/config.perfect.yaml +++ b/config/test/config.perfect.yaml @@ -75,8 +75,8 @@ industry: solving: solver: - name: glpk - options: glpk-default + name: highs + options: highs-default mem: 4000 plotting: diff --git a/doc/release_notes.rst b/doc/release_notes.rst index c20f1ebe..55e0ad97 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -62,6 +62,11 @@ Upcoming Release * To determine the optimal topology to meet the number of clusters, the workflow used pyomo in combination with ``ipopt`` or ``gurobi``. This dependency has been replaced by using ``linopy`` in combination with ``scipopt`` or ``gurobi``. The environment file has been updated accordingly. +* The ``highs`` solver was added to the default environment file. + +* The default solver for testing the workflow in the CI has been changed from ``glpk`` to ``highs``. + + PyPSA-Eur 0.9.0 (5th January 2024) ================================== diff --git a/envs/environment.yaml b/envs/environment.yaml index d92eb696..235d7729 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -59,3 +59,4 @@ dependencies: - pip: - tsam>=2.3.1 + - highspy From 718dbbb1c50ba25fb004925e993e517c2e8dc10e Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 31 Jan 2024 09:54:58 +0100 Subject: [PATCH 08/15] CI: integrate inhouse tests into main CI yaml --- .github/workflows/ci-sync.yaml | 97 ---------------------------------- .github/workflows/ci.yaml | 14 ++++- 2 files changed, 13 insertions(+), 98 deletions(-) delete mode 100644 .github/workflows/ci-sync.yaml diff --git a/.github/workflows/ci-sync.yaml b/.github/workflows/ci-sync.yaml deleted file mode 100644 index e5b4006c..00000000 --- a/.github/workflows/ci-sync.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors -# -# SPDX-License-Identifier: CC0-1.0 - -name: CI - Synced Inhouse Packages - -# Caching method based on and described by: -# epassaro (2021): https://dev.to/epassaro/caching-anaconda-environments-in-github-actions-5hde -# and code in GitHub repo: https://github.com/epassaro/cache-conda-envs - -on: - push: - branches: - - master - pull_request: - branches: - - master - schedule: - - cron: "0 5 * * TUE" - -env: - DATA_CACHE_NUMBER: 2 - -jobs: - build: - - strategy: - fail-fast: false - max-parallel: 1 - - runs-on: ubuntu-latest - - defaults: - run: - shell: bash -l {0} - - steps: - - uses: actions/checkout@v3 - - - name: Setup secrets - run: | - echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc - - - name: Add solver to environment - run: | - echo -e "- glpk\n- ipopt<3.13.3" >> envs/environment.yaml - if: ${{ matrix.os }} == 'windows-latest' - - - name: Add solver to environment - run: | - echo -e "- glpk\n- ipopt" >> envs/environment.yaml - if: ${{ matrix.os }} != 'windows-latest' - - - name: Setup micromamba - uses: mamba-org/setup-micromamba@v1 - with: - micromamba-version: latest - environment-file: envs/environment.yaml - log-level: debug - init-shell: bash - cache-environment: true - cache-downloads: true - - # install inhouse packages from master branch with pip - - name: Install inhouse packages - run: | - pip install git+https://github.com/PyPSA/atlite.git@master git+https://github.com/PyPSA/powerplantmatching.git@master git+https://github.com/PyPSA/linopy.git@master - - - - name: Set cache dates - run: | - echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV - - - name: Cache data and cutouts folders - uses: actions/cache@v3 - with: - path: | - data - cutouts - key: data-cutouts-${{ env.WEEK }}-${{ env.DATA_CACHE_NUMBER }} - - - name: Test snakemake workflow - run: | - snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime - snakemake -call all --configfile config/test/config.overnight.yaml --rerun-triggers=mtime - snakemake -call all --configfile config/test/config.myopic.yaml --rerun-triggers=mtime - snakemake -call all --configfile config/test/config.perfect.yaml --rerun-triggers=mtime - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: resources-results - path: | - resources - results - if-no-files-found: warn - retention-days: 1 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c0fb745d..172d07ac 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -32,7 +32,14 @@ jobs: - ubuntu-latest - macos-latest - windows-latest - + inhouse: + - stable + - master + exclude: + - os: macos-latest + inhouse: master + - os: windows-latest + inhouse: master runs-on: ${{ matrix.os }} defaults: @@ -66,6 +73,11 @@ jobs: cache-environment: true cache-downloads: true + - name: Install inhouse packages + run: | + pip install git+https://github.com/PyPSA/atlite.git@master git+https://github.com/PyPSA/powerplantmatching.git@master git+https://github.com/PyPSA/linopy.git@master + if: ${{ matrix.inhouse }} == 'master' + - name: Set cache dates run: | echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV From c4a634659dc008f9e10095e8787e11064a42ae33 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 31 Jan 2024 09:59:31 +0100 Subject: [PATCH 09/15] CI: correct if statement --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 172d07ac..637a0304 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -76,7 +76,7 @@ jobs: - name: Install inhouse packages run: | pip install git+https://github.com/PyPSA/atlite.git@master git+https://github.com/PyPSA/powerplantmatching.git@master git+https://github.com/PyPSA/linopy.git@master - if: ${{ matrix.inhouse }} == 'master' + if: ${{ matrix.inhouse }} == 'master' - name: Set cache dates run: | From df2296ce3effcb953411892ec6abfc70ea2410e1 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 31 Jan 2024 12:23:35 +0100 Subject: [PATCH 10/15] ci: only upload artefacts for ubuntu and stable --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 637a0304..bba03cd7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -98,7 +98,7 @@ jobs: snakemake -call all --configfile config/test/config.perfect.yaml --rerun-triggers=mtime - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4.3.0 with: name: resources-results path: | @@ -106,3 +106,4 @@ jobs: results if-no-files-found: warn retention-days: 1 + if: matrix.os == 'ubuntu' && matrix.inhouse == 'stable' From bd55e368f7510aacfc234def4af4b525c4b740ab Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 31 Jan 2024 13:04:10 +0100 Subject: [PATCH 11/15] test: revert setting highs as default solver, as not available for macos --- config/test/config.electricity.yaml | 4 ++-- config/test/config.myopic.yaml | 4 ++-- config/test/config.overnight.yaml | 4 ++-- config/test/config.perfect.yaml | 4 ++-- doc/release_notes.rst | 2 -- envs/environment.yaml | 1 + 6 files changed, 9 insertions(+), 10 deletions(-) diff --git a/config/test/config.electricity.yaml b/config/test/config.electricity.yaml index ef9bfdc0..b750bf62 100644 --- a/config/test/config.electricity.yaml +++ b/config/test/config.electricity.yaml @@ -69,8 +69,8 @@ lines: solving: solver: - name: highs - options: "highs-default" + name: glpk + options: "glpk-default" plotting: diff --git a/config/test/config.myopic.yaml b/config/test/config.myopic.yaml index 1da51b36..2dab7b04 100644 --- a/config/test/config.myopic.yaml +++ b/config/test/config.myopic.yaml @@ -73,8 +73,8 @@ industry: solving: solver: - name: highs - options: highs-default + name: glpk + options: glpk-default mem: 4000 plotting: diff --git a/config/test/config.overnight.yaml b/config/test/config.overnight.yaml index dffdab5a..6d1900cf 100644 --- a/config/test/config.overnight.yaml +++ b/config/test/config.overnight.yaml @@ -65,8 +65,8 @@ sector: solving: solver: - name: highs - options: highs-default + name: glpk + options: glpk-default mem: 4000 plotting: diff --git a/config/test/config.perfect.yaml b/config/test/config.perfect.yaml index 8a5a9cce..f20a2c9f 100644 --- a/config/test/config.perfect.yaml +++ b/config/test/config.perfect.yaml @@ -75,8 +75,8 @@ industry: solving: solver: - name: highs - options: highs-default + name: glpk + options: glpk-default mem: 4000 plotting: diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 55e0ad97..6fa88738 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -64,8 +64,6 @@ Upcoming Release * The ``highs`` solver was added to the default environment file. -* The default solver for testing the workflow in the CI has been changed from ``glpk`` to ``highs``. - PyPSA-Eur 0.9.0 (5th January 2024) diff --git a/envs/environment.yaml b/envs/environment.yaml index 235d7729..26e18f0d 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -35,6 +35,7 @@ dependencies: - netcdf4 - networkx - scipy +- glpk - shapely>=2.0 - pyscipopt - matplotlib From 550e6f07940399b3c4ee6ee3cf0c274981d4aecf Mon Sep 17 00:00:00 2001 From: JulianGeis Date: Thu, 1 Feb 2024 14:23:04 +0100 Subject: [PATCH 12/15] deleted code to include kernnetz as it is no longer needed --- scripts/prepare_sector_network.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 42def055..03cba48b 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1319,22 +1319,6 @@ def add_storage_and_grids(n, costs): n, "H2 pipeline ", carriers=["DC", "gas pipeline"] ) - h2_pipes["p_nom"] = 0.0 - - if snakemake.input.get("custom_h2_pipelines"): - fn = snakemake.input.custom_h2_pipelines - custom_pipes = pd.read_csv(fn, index_col=0) - - h2_pipes = pd.concat([h2_pipes, custom_pipes]) - - # drop duplicates according to buses (order can be different) and keep pipe with highest p_nom - h2_pipes["buses_sorted"] = h2_pipes[["bus0", "bus1"]].apply(sorted, axis=1) - h2_pipes = ( - h2_pipes.sort_values("p_nom") - .drop_duplicates(subset=["buses_sorted"], keep="last") - .drop(columns="buses_sorted") - ) - # TODO Add efficiency losses n.madd( "Link", @@ -1343,7 +1327,6 @@ def add_storage_and_grids(n, costs): bus1=h2_pipes.bus1.values + " H2", p_min_pu=-1, p_nom_extendable=True, - p_nom_min=h2_pipes.p_nom.values, length=h2_pipes.length.values, capital_cost=costs.at["H2 (g) pipeline", "fixed"] * h2_pipes.length.values, carrier="H2 pipeline", From cc77456195b5264db4b8a823499b83d180b4a88e Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 2 Feb 2024 10:11:59 +0100 Subject: [PATCH 13/15] env: add pre-commit --- envs/environment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/envs/environment.yaml b/envs/environment.yaml index 26e18f0d..88e394fa 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -48,6 +48,7 @@ dependencies: - tabula-py - pyxlsb - graphviz +- pre-commit # Keep in conda environment when calling ipython - ipython From 6d40ff1340898bc2b5b61c5630cebe9ae72f9b0a Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 2 Feb 2024 11:52:01 +0100 Subject: [PATCH 14/15] config: add sensible defaults for HVC recycling and reuse --- config/config.default.yaml | 31 ++++++++++++++++++++++++++++--- doc/release_notes.rst | 6 ++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 51080862..5afc6f8e 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -606,9 +606,34 @@ industry: MWh_NH3_per_MWh_H2_cracker: 1.46 # https://github.com/euronion/trace/blob/44a5ff8401762edbef80eff9cfe5a47c8d3c8be4/data/efficiencies.csv NH3_process_emissions: 24.5 petrochemical_process_emissions: 25.5 - HVC_primary_fraction: 1. - HVC_mechanical_recycling_fraction: 0. - HVC_chemical_recycling_fraction: 0. + #HVC primary/recycling based on values used in Neumann et al https://doi.org/10.1016/j.joule.2023.06.016, linearly interpolated between 2020 and 2050 + #2020 recycling rates based on Agora https://static.agora-energiewende.de/fileadmin/Projekte/2021/2021_02_EU_CEAP/A-EW_254_Mobilising-circular-economy_study_WEB.pdf + #fractions refer to the total primary HVC production in 2020 + #assumes 6.7 Mtplastics produced from recycling in 2020 + HVC_primary_fraction: + 2020: 1.0 + 2025: 0.9 + 2030: 0.8 + 2035: 0.7 + 2040: 0.6 + 2045: 0.5 + 2050: 0.4 + HVC_mechanical_recycling_fraction: + 2020: 0.12 + 2025: 0.15 + 2030: 0.18 + 2035: 0.21 + 2040: 0.24 + 2045: 0.27 + 2050: 0.30 + HVC_chemical_recycling_fraction: + 2020: 0.0 + 2025: 0.0 + 2030: 0.04 + 2035: 0.08 + 2040: 0.12 + 2045: 0.16 + 2050: 0.20 HVC_production_today: 52. MWh_elec_per_tHVC_mechanical_recycling: 0.547 MWh_elec_per_tHVC_chemical_recycling: 6.9 diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 6fa88738..a3331748 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -64,6 +64,12 @@ Upcoming Release * The ``highs`` solver was added to the default environment file. +* Default settings for recycling rates and primary product shares of high-value + chemicals have been set in accordance with the values used in `Neumann et al. + (2023) `_ linearly interpolated + between 2020 and 2050. The recycling rates are based on data from `Agora + Energiewende (2021) + `_. PyPSA-Eur 0.9.0 (5th January 2024) From 0006f6e5a90b548d80cc448711edab145470917e Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Fri, 2 Feb 2024 12:19:08 +0000 Subject: [PATCH 15/15] Fix gas input for existing heat gas boilers --- scripts/add_existing_baseyear.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 46d4bc31..ca11a747 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -482,7 +482,7 @@ def add_heating_capacities_installed_before_baseyear( "Link", nodes, suffix=f" {name} gas boiler-{grouping_year}", - bus0=spatial.gas.nodes, + bus0="EU gas" if "EU gas" in spatial.gas.nodes else nodes + " gas", bus1=nodes + " " + name + " heat", bus2="co2 atmosphere", carrier=name + " gas boiler",