ci: better inhouse checks (#1331)

* ci: better inhouse checks

* fix: no uv

.github/workflows/test.yaml

@@ -19,9 +19,12 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  ENV_FILE: envs/environment.yaml
+
 jobs:
-  run:
+  run-tests:
-    name: Run
+    name: OS
     strategy:
       fail-fast: false
       matrix:
@@ -29,14 +32,6 @@ jobs:
         - macos
         - windows
         - ubuntu
-        inhouse:
-        - stable-inhouse-deps
-        - dev-inhouse-deps
-        exclude:
-        - os: macos
-          inhouse: dev-inhouse-deps
-        - os: windows
-          inhouse: dev-inhouse-deps
     runs-on: ${{ matrix.os }}-latest
 
     defaults:
@@ -46,25 +41,20 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Setup secrets
+    - name: Setup secrets & cache dates
       run: |
         echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc
-
-    - name: Set cache dates
-      run: |
         echo "week=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
         echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_ENV # env
 
-    - name: Cache data and cutouts folders
+    - uses: actions/cache@v4
-      uses: actions/cache@v4
       with:
         path: |
           data
           cutouts
         key: data-cutouts-${{ env.week }}
 
-    - name: Setup Conda
+    - uses: conda-incubator/setup-miniconda@v3
-      uses: conda-incubator/setup-miniconda@v3
       with:
         activate-environment: pypsa-eur
 
@@ -72,31 +62,105 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ${{ env.CONDA }}/envs
-        key: conda-${{ runner.os }}--${{ runner.arch }}--${{ env.today }}-${{ hashFiles('envs/environment.yaml') }}
+        key: conda-${{ runner.os }}-${{ runner.arch }}-${{ env.today }}-${hashFiles('${{ env.ENV_FILE }}')}
       id: cache-env
 
     - name: Update environment
       if: steps.cache-env.outputs.cache-hit != 'true'
-      run: conda env update -n pypsa-eur -f envs/environment.yaml
+      run: conda env update -n pypsa-eur -f ${{ env.ENV_FILE }}
-
-    - name: Install inhouse packages from master
-      if: matrix.inhouse == 'dev-inhouse-deps'
-      run: |
-        python -m pip install uv
-        uv pip install git+https://github.com/PyPSA/pypsa.git@master
-        uv pip install git+https://github.com/PyPSA/atlite.git@master
-        uv pip install git+https://github.com/PyPSA/powerplantmatching.git@master
-        uv pip install git+https://github.com/PyPSA/linopy.git@master
 
     - name: Run snakemake test workflows
       run: |
+        conda activate pypsa-eur
         make test
 
     - name: Upload artifacts
-      if: matrix.os == 'ubuntu' && matrix.inhouse == 'stable-inhouse-deps'
+      uses: actions/upload-artifact@v3.2.1-node20
-      uses: actions/upload-artifact@v4.4.0
       with:
-        name: resources-results
+        name: logs-${{ matrix.inhouse }}
         path: |
-          results
+          logs
-        retention-days: 7
+          .snakemake/log
+        retention-days: 3
+
+  run-tests-on-dev-deps:
+    name: Inhouse
+    strategy:
+      fail-fast: false
+      matrix:
+        inhouse:
+        - pypsa
+        - atlite
+        - powerplantmatching
+        - linopy
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+    - uses: actions/checkout@v4
+
+    # Only run checks if package is not pinned
+    - name: Check if inhouse package is pinned
+      run: |
+        grep_line=$(grep -- '- ${{ matrix.inhouse }}' ${{ env.ENV_FILE }})
+        if [[ $grep_line == *"<"* || $grep_line == *"=="* ]]; then
+            echo "pinned=true" >> $GITHUB_ENV
+        else
+            echo "pinned=false" >> $GITHUB_ENV
+        fi
+
+    - name: Setup secrets & cache dates
+      if: env.pinned == 'false'
+      run: |
+        echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc
+        echo "week=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
+        echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_ENV # env
+
+    - uses: actions/cache@v4
+      if: env.pinned == 'false'
+      with:
+        path: |
+          data
+          cutouts
+        key: data-cutouts-${{ env.week }}
+
+    - uses: conda-incubator/setup-miniconda@v3
+      if: env.pinned == 'false'
+      with:
+        activate-environment: pypsa-eur
+
+    - name: Cache Conda env
+      if: env.pinned == 'false'
+      uses: actions/cache@v4
+      with:
+        path: ${{ env.CONDA }}/envs
+        key: conda-${{ runner.os }}-${{ runner.arch }}-${{  matrix.inhouse  }}-${{ env.today }}-${hashFiles('${{ env.ENV_FILE }}')}
+      id: cache-env
+
+    - name: Update environment
+      if: env.pinned == 'false' && steps.cache-env.outputs.cache-hit != 'true'
+      run: conda env update -n pypsa-eur -f ${{ env.ENV_FILE }}
+
+    - name: Install inhouse packages from master
+      if: env.pinned == 'false'
+      run: |
+        python -m pip install git+https://github.com/PyPSA/${{ matrix.inhouse }}.git@master
+
+    - name: Run snakemake test workflows
+      if: env.pinned == 'false'
+      run: |
+        conda activate pypsa-eur
+        make test
+
+    - name: Upload artifacts
+      if: env.pinned == 'false'
+      uses: actions/upload-artifact@v4
+      with:
+        name: logs-${{ matrix.inhouse }}
+        path: |
+          logs
+          .snakemake/log
+        retention-days: 3

config/test/config.electricity.yaml

@@ -79,8 +79,8 @@ lines:
 
 solving:
   solver:
-    name: glpk
+    name: highs
-    options: "glpk-default"
+    options: highs-default
 
 
 plotting:

config/test/config.myopic.yaml

@@ -86,8 +86,8 @@ industry:
 
 solving:
   solver:
-    name: glpk
+    name: highs
-    options: glpk-default
+    options: highs-default
   mem: 4000
 
 plotting:

config/test/config.overnight.yaml

@@ -80,8 +80,8 @@ industry:
 
 solving:
   solver:
-    name: glpk
+    name: highs
-    options: glpk-default
+    options: highs-default
   mem: 4000
 
 plotting:

config/test/config.perfect.yaml

@@ -85,8 +85,8 @@ industry:
 
 solving:
   solver:
-    name: glpk
+    name: highs
-    options: glpk-default
+    options: highs-default
   mem: 4000
 
 plotting:

config/test/config.scenarios.yaml

@@ -57,5 +57,5 @@ renewable:
 
 solving:
   solver:
-    name: glpk
+    name: highs
-    options: "glpk-default"
+    options: highs-default