test: fix and update test workflows (#1296)

* test: fix and update test workflows

* resolve conflict

* bring back test.sh

.github/workflows/ci.yaml

@@ -1,95 +0,0 @@
-# SPDX-FileCopyrightText: : 2021-2024 The PyPSA-Eur Authors
-#
-# SPDX-License-Identifier: CC0-1.0
-
-name: CI
-
-# Caching method based on and described by:
-# epassaro (2021): https://dev.to/epassaro/caching-anaconda-environments-in-github-actions-5hde
-# and code in GitHub repo: https://github.com/epassaro/cache-conda-envs
-
-on:
-  push:
-    branches:
-    - master
-  pull_request:
-    branches:
-    - master
-  schedule:
-  - cron: "0 5 * * TUE"
-
-env:
-  DATA_CACHE_NUMBER: 2
-
-jobs:
-  build:
-
-    strategy:
-      fail-fast: false
-      max-parallel: 3
-      matrix:
-        os:
-        - ubuntu-latest
-        - macos-latest
-        - windows-latest
-        inhouse:
-        - stable
-        - master
-        exclude:
-        - os: macos-latest
-          inhouse: master
-        - os: windows-latest
-          inhouse: master
-    runs-on: ${{ matrix.os }}
-
-    defaults:
-      run:
-        shell: bash -l {0}
-
-    steps:
-    - uses: actions/checkout@v4
-
-    - name: Setup secrets
-      run: |
-        echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc
-
-    - name: Setup micromamba
-      uses: mamba-org/setup-micromamba@v1
-      with:
-        micromamba-version: latest
-        environment-file: envs/environment.yaml
-        log-level: debug
-        init-shell: bash
-        cache-environment: true
-        cache-downloads: true
-
-    - name: Install inhouse packages
-      run: |
-        pip install git+https://github.com/PyPSA/atlite.git@master git+https://github.com/PyPSA/powerplantmatching.git@master git+https://github.com/PyPSA/linopy.git@master
-      if: ${{ matrix.inhouse }} == 'master'
-
-    - name: Set cache dates
-      run: |
-        echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV
-
-    - name: Cache data and cutouts folders
-      uses: actions/cache@v4
-      with:
-        path: |
-          data
-          cutouts
-        key: data-cutouts-${{ env.WEEK }}-${{ env.DATA_CACHE_NUMBER }}
-
-    - name: Test snakemake workflow
-      run: ./test.sh
-
-    - name: Upload artifacts
-      uses: actions/upload-artifact@v4.4.0
-      with:
-        name: resources-results
-        path: |
-          resources
-          results
-        if-no-files-found: warn
-        retention-days: 1
-      if: matrix.os == 'ubuntu' && matrix.inhouse == 'stable'

.github/workflows/test.yaml

@@ -0,0 +1,112 @@
+# SPDX-FileCopyrightText: : 2021-2024 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: CC0-1.0
+
+name: Test workflows
+
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+    branches:
+    - master
+  schedule:
+  - cron: "0 5 * * TUE"
+
+# Cancel any in-progress runs when a new run is triggered
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run:
+    name: Run
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+        - macos
+        - windows
+        - ubuntu
+        inhouse:
+        - stable-inhouse-deps
+        - dev-inhouse-deps
+        exclude:
+        - os: macos
+          inhouse: dev-inhouse-deps
+        - os: windows
+          inhouse: dev-inhouse-deps
+    runs-on: ${{ matrix.os }}-latest
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Setup secrets
+      run: |
+        echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc
+
+    - name: Set cache dates
+      run: |
+        echo "week=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
+        echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_ENV # env
+
+    - name: Cache data and cutouts folders
+      uses: actions/cache@v4
+      with:
+        path: |
+          data
+          cutouts
+        key: data-cutouts-${{ env.week }}
+
+    - name: Setup Conda
+      uses: conda-incubator/setup-miniconda@v3
+      with:
+        activate-environment: pypsa-eur
+
+    - name: Cache Conda env
+      uses: actions/cache@v3
+      with:
+        path: ${{ env.CONDA }}/envs
+        key: conda-${{ runner.os }}--${{ runner.arch }}--${{ env.today }}-${{ hashFiles('envs/environment.yaml') }}
+      id: cache-env
+
+    - name: Update environment
+      if: steps.cache-env.outputs.cache-hit != 'true' && matrix.os != 'macos'
+      run: conda env update -n pypsa-eur -f envs/environment.yaml
+
+    # Temporary fix for MacOS, since highspy with pypsa can not be resolved
+    - name: Update environment (macos specific)
+      if: steps.cache-env.outputs.cache-hit != 'true' && matrix.os == 'macos'
+      run: |
+        sed -i '' '/- pypsa/d' envs/environment.yaml # Remove pypsa from environment list
+        conda env update -n pypsa-eur -f envs/environment.yaml
+        pip install highspy
+        conda install -c conda-forge pypsa
+
+    - name: Install inhouse packages from master
+      if: matrix.inhouse == 'dev-inhouse-deps'
+      run: |
+        python -m pip install uv
+        uv pip install git+https://github.com/PyPSA/pypsa.git@master
+        uv pip install git+https://github.com/PyPSA/atlite.git@master
+        uv pip install git+https://github.com/PyPSA/powerplantmatching.git@master
+        uv pip install git+https://github.com/PyPSA/linopy.git@master
+
+    - name: Run snakemake test workflows
+      run: |
+        chmod +x test.sh
+        ./test.sh
+
+    - name: Upload artifacts
+      if: matrix.os == 'ubuntu' && matrix.inhouse == 'stable-inhouse-deps'
+      uses: actions/upload-artifact@v4.4.0
+      with:
+        name: resources-results
+        path: |
+          results
+        retention-days: 7

README.md

@@ -4,7 +4,7 @@ SPDX-License-Identifier: CC-BY-4.0
 -->
 
 ![GitHub release (latest by date including pre-releases)](https://img.shields.io/github/v/release/pypsa/pypsa-eur?include_prereleases)
-[![Build Status](https://github.com/pypsa/pypsa-eur/actions/workflows/ci.yaml/badge.svg)](https://github.com/PyPSA/pypsa-eur/actions)
+[![Test workflows](https://github.com/pypsa/pypsa-eur/actions/workflows/test.yaml/badge.svg)](https://github.com/pypsa/pypsa-eur/actions/workflows/test.yaml)
 [![Documentation](https://readthedocs.org/projects/pypsa-eur/badge/?version=latest)](https://pypsa-eur.readthedocs.io/en/latest/?badge=latest)
 ![Size](https://img.shields.io/github/repo-size/pypsa/pypsa-eur)
 [![Zenodo PyPSA-Eur](https://zenodo.org/badge/DOI/10.5281/zenodo.3520874.svg)](https://doi.org/10.5281/zenodo.3520874)

doc/index.rst

@@ -10,7 +10,7 @@ PyPSA-Eur: A Sector-Coupled Open Optimisation Model of the European Energy Syste
 .. image:: https://img.shields.io/github/v/release/pypsa/pypsa-eur?include_prereleases
     :alt: GitHub release (latest by date including pre-releases)
 
-.. image:: https://github.com/pypsa/pypsa-eur/actions/workflows/ci.yaml/badge.svg
+.. image:: https://github.com/pypsa/pypsa-eur/actions/workflows/test.yaml/badge.svg
     :target: https://github.com/PyPSA/pypsa-eur/actions
 
 .. image:: https://readthedocs.org/projects/pypsa-eur/badge/?version=latest

envs/environment.yaml

@@ -15,7 +15,7 @@ dependencies:
 - linopy
 - dask
 
-  # Dependencies of the workflow itself
+# Dependencies of the workflow itself
 - xlrd
 - openpyxl!=3.1.1
 - seaborn
@@ -49,15 +49,14 @@ dependencies:
 - pre-commit
 - geojson
 
-  # Keep in conda environment when calling ipython
+# Keep in conda environment when calling ipython
 - ipython
 
-  # GIS dependencies:
+# GIS dependencies:
 - cartopy
 - descartes
 - rasterio!=1.2.10
 
-
 - pip:
   - tsam>=2.3.1
   - snakemake-storage-plugin-http