Merge pull request #303 from PyPSA/misc/improve-ci-speed

[DNMY] Improve CI performance.

.github/workflows/ci.yaml

@@ -4,6 +4,10 @@
 
 name: CI
 
+# Caching method based on and described by:
+# epassaro (2021): https://dev.to/epassaro/caching-anaconda-environments-in-github-actions-5hde
+# and code in GitHub repo: https://github.com/epassaro/cache-conda-envs
+
 on:
   push:
     branches:
@@ -14,42 +18,73 @@ on:
   schedule:
     - cron: "0 5 * * TUE"
 
+env:
+  CACHE_NUMBER: 1 # Change this value to manually reset the environment cache
+
 jobs:
   build:
 
-    runs-on: ${{ matrix.os }}
     strategy:
-      max-parallel: 5
       matrix:
-        os:
+        include:
-        - ubuntu-latest
+          # Matrix required to handle caching with Mambaforge
-        - macos-latest
+          - os: ubuntu-latest
-        - windows-latest
+            label: ubuntu-latest
+            prefix: /usr/share/miniconda3/envs/pypsa-eur
+
+          - os: macos-latest
+            label: macos-latest
+            prefix: /Users/runner/miniconda3/envs/pypsa-eur
+
+          - os: windows-latest
+            label: windows-latest
+            prefix: C:\Miniconda3\envs\pypsa-eur
+
+    name: ${{ matrix.label }}
+
+    runs-on: ${{ matrix.os }}
     
     defaults:
       run:
         shell: bash -l {0}
         
     steps:
+      - uses: actions/checkout@v2
       
-    - uses: actions/checkout@v2
+      - name: Setup secrets
+        run: |
+          echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc
 
-    - name: Setup Miniconda
+      - name: Add solver to environment
-      uses: conda-incubator/setup-miniconda@v2.1.1 
+        run: |
-      with: # checks out environment 'test' by default
+          echo -e "  - glpk\n  - ipopt<3.13.3" >> envs/environment.yaml
-          mamba-version: "*"
-          channels: conda-forge,defaults
-          channel-priority: true
 
-    - name: Install dependencies  
+      - name: Setup Mambaforge
-      run: |
+        uses: conda-incubator/setup-miniconda@v2
-        echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc
+        with:
-        echo -e "  - glpk\n  - ipopt<3.13.3" >> envs/environment.yaml
+            miniforge-variant: Mambaforge
-        mamba env update -f envs/environment.yaml --name test
+            miniforge-version: latest
+            activate-environment: pypsa-eur
+            use-mamba: true
             
-    - name: Test snakemake workflow
+      - name: Set cache date
-      run: |
+        run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV
-        conda list
+
-        cp test/config.test1.yaml config.yaml
+      - name: Create environment cache
-        snakemake --cores all solve_all_networks
+        uses: actions/cache@v2
-        rm -rf resources/*.nc resources/*.geojson resources/*.h5 networks results
+        id: cache
+        with:
+          path: ${{ matrix.prefix }}
+          key: ${{ matrix.label }}-conda-${{ hashFiles('envs/environment.yaml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
+
+      - name: Update environment due to outdated or unavailable cache
+        run: mamba env update -n pypsa-eur -f envs/environment.yaml
+        if: steps.cache.outputs.cache-hit != 'true'
+
+      - name: Test snakemake workflow
+        run: |
+          conda activate pypsa-eur
+          conda list
+          cp test/config.test1.yaml config.yaml
+          snakemake --cores all solve_all_networks
+          rm -rf resources/*.nc resources/*.geojson resources/*.h5 networks results

envs/environment.yaml

@@ -6,11 +6,9 @@ name: pypsa-eur
 channels:
   - conda-forge
   - bioconda
-  - http://conda.anaconda.org/gurobi
 dependencies:
   - python>=3.8
   - pip
-  - mamba # esp for windows build
 
   - pypsa>=0.18
   - atlite>=0.2.5