From 08a4d196af265ba1e4fc5729b33f654cdd8ffc7b Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 26 Jan 2024 23:16:33 +0100 Subject: [PATCH 1/4] ci: add workflow with inhouse package at current master --- .github/workflows/ci-sync.yaml | 97 ++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 .github/workflows/ci-sync.yaml diff --git a/.github/workflows/ci-sync.yaml b/.github/workflows/ci-sync.yaml new file mode 100644 index 00000000..e5b4006c --- /dev/null +++ b/.github/workflows/ci-sync.yaml @@ -0,0 +1,97 @@ +# SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC0-1.0 + +name: CI - Synced Inhouse Packages + +# Caching method based on and described by: +# epassaro (2021): https://dev.to/epassaro/caching-anaconda-environments-in-github-actions-5hde +# and code in GitHub repo: https://github.com/epassaro/cache-conda-envs + +on: + push: + branches: + - master + pull_request: + branches: + - master + schedule: + - cron: "0 5 * * TUE" + +env: + DATA_CACHE_NUMBER: 2 + +jobs: + build: + + strategy: + fail-fast: false + max-parallel: 1 + + runs-on: ubuntu-latest + + defaults: + run: + shell: bash -l {0} + + steps: + - uses: actions/checkout@v3 + + - name: Setup secrets + run: | + echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc + + - name: Add solver to environment + run: | + echo -e "- glpk\n- ipopt<3.13.3" >> envs/environment.yaml + if: ${{ matrix.os }} == 'windows-latest' + + - name: Add solver to environment + run: | + echo -e "- glpk\n- ipopt" >> envs/environment.yaml + if: ${{ matrix.os }} != 'windows-latest' + + - name: Setup micromamba + uses: mamba-org/setup-micromamba@v1 + with: + micromamba-version: latest + environment-file: envs/environment.yaml + log-level: debug + init-shell: bash + cache-environment: true + cache-downloads: true + + # install inhouse packages from master branch with pip + - name: Install inhouse packages + run: | + pip install git+https://github.com/PyPSA/atlite.git@master git+https://github.com/PyPSA/powerplantmatching.git@master git+https://github.com/PyPSA/linopy.git@master + + + - name: Set cache dates + run: | + echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV + + - name: Cache data and cutouts folders + uses: actions/cache@v3 + with: + path: | + data + cutouts + key: data-cutouts-${{ env.WEEK }}-${{ env.DATA_CACHE_NUMBER }} + + - name: Test snakemake workflow + run: | + snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime + snakemake -call all --configfile config/test/config.overnight.yaml --rerun-triggers=mtime + snakemake -call all --configfile config/test/config.myopic.yaml --rerun-triggers=mtime + snakemake -call all --configfile config/test/config.perfect.yaml --rerun-triggers=mtime + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: resources-results + path: | + resources + results + if-no-files-found: warn + retention-days: 1 From 718dbbb1c50ba25fb004925e993e517c2e8dc10e Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 31 Jan 2024 09:54:58 +0100 Subject: [PATCH 2/4] CI: integrate inhouse tests into main CI yaml --- .github/workflows/ci-sync.yaml | 97 ---------------------------------- .github/workflows/ci.yaml | 14 ++++- 2 files changed, 13 insertions(+), 98 deletions(-) delete mode 100644 .github/workflows/ci-sync.yaml diff --git a/.github/workflows/ci-sync.yaml b/.github/workflows/ci-sync.yaml deleted file mode 100644 index e5b4006c..00000000 --- a/.github/workflows/ci-sync.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# SPDX-FileCopyrightText: : 2021-2023 The PyPSA-Eur Authors -# -# SPDX-License-Identifier: CC0-1.0 - -name: CI - Synced Inhouse Packages - -# Caching method based on and described by: -# epassaro (2021): https://dev.to/epassaro/caching-anaconda-environments-in-github-actions-5hde -# and code in GitHub repo: https://github.com/epassaro/cache-conda-envs - -on: - push: - branches: - - master - pull_request: - branches: - - master - schedule: - - cron: "0 5 * * TUE" - -env: - DATA_CACHE_NUMBER: 2 - -jobs: - build: - - strategy: - fail-fast: false - max-parallel: 1 - - runs-on: ubuntu-latest - - defaults: - run: - shell: bash -l {0} - - steps: - - uses: actions/checkout@v3 - - - name: Setup secrets - run: | - echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc - - - name: Add solver to environment - run: | - echo -e "- glpk\n- ipopt<3.13.3" >> envs/environment.yaml - if: ${{ matrix.os }} == 'windows-latest' - - - name: Add solver to environment - run: | - echo -e "- glpk\n- ipopt" >> envs/environment.yaml - if: ${{ matrix.os }} != 'windows-latest' - - - name: Setup micromamba - uses: mamba-org/setup-micromamba@v1 - with: - micromamba-version: latest - environment-file: envs/environment.yaml - log-level: debug - init-shell: bash - cache-environment: true - cache-downloads: true - - # install inhouse packages from master branch with pip - - name: Install inhouse packages - run: | - pip install git+https://github.com/PyPSA/atlite.git@master git+https://github.com/PyPSA/powerplantmatching.git@master git+https://github.com/PyPSA/linopy.git@master - - - - name: Set cache dates - run: | - echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV - - - name: Cache data and cutouts folders - uses: actions/cache@v3 - with: - path: | - data - cutouts - key: data-cutouts-${{ env.WEEK }}-${{ env.DATA_CACHE_NUMBER }} - - - name: Test snakemake workflow - run: | - snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime - snakemake -call all --configfile config/test/config.overnight.yaml --rerun-triggers=mtime - snakemake -call all --configfile config/test/config.myopic.yaml --rerun-triggers=mtime - snakemake -call all --configfile config/test/config.perfect.yaml --rerun-triggers=mtime - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: resources-results - path: | - resources - results - if-no-files-found: warn - retention-days: 1 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c0fb745d..172d07ac 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -32,7 +32,14 @@ jobs: - ubuntu-latest - macos-latest - windows-latest - + inhouse: + - stable + - master + exclude: + - os: macos-latest + inhouse: master + - os: windows-latest + inhouse: master runs-on: ${{ matrix.os }} defaults: @@ -66,6 +73,11 @@ jobs: cache-environment: true cache-downloads: true + - name: Install inhouse packages + run: | + pip install git+https://github.com/PyPSA/atlite.git@master git+https://github.com/PyPSA/powerplantmatching.git@master git+https://github.com/PyPSA/linopy.git@master + if: ${{ matrix.inhouse }} == 'master' + - name: Set cache dates run: | echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV From c4a634659dc008f9e10095e8787e11064a42ae33 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 31 Jan 2024 09:59:31 +0100 Subject: [PATCH 3/4] CI: correct if statement --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 172d07ac..637a0304 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -76,7 +76,7 @@ jobs: - name: Install inhouse packages run: | pip install git+https://github.com/PyPSA/atlite.git@master git+https://github.com/PyPSA/powerplantmatching.git@master git+https://github.com/PyPSA/linopy.git@master - if: ${{ matrix.inhouse }} == 'master' + if: ${{ matrix.inhouse }} == 'master' - name: Set cache dates run: | From df2296ce3effcb953411892ec6abfc70ea2410e1 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 31 Jan 2024 12:23:35 +0100 Subject: [PATCH 4/4] ci: only upload artefacts for ubuntu and stable --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 637a0304..bba03cd7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -98,7 +98,7 @@ jobs: snakemake -call all --configfile config/test/config.perfect.yaml --rerun-triggers=mtime - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4.3.0 with: name: resources-results path: | @@ -106,3 +106,4 @@ jobs: results if-no-files-found: warn retention-days: 1 + if: matrix.os == 'ubuntu' && matrix.inhouse == 'stable'