From 991916370f810a5588b2788ac9a301221e125de3 Mon Sep 17 00:00:00 2001 From: Lukas Trippe Date: Fri, 27 Sep 2024 13:07:02 +0200 Subject: [PATCH] ci: better inhouse checks (#1331) * ci: better inhouse checks * fix: no uv --- .github/workflows/test.yaml | 132 +++++++++++++++++++++------- config/test/config.electricity.yaml | 4 +- config/test/config.myopic.yaml | 4 +- config/test/config.overnight.yaml | 4 +- config/test/config.perfect.yaml | 4 +- config/test/config.scenarios.yaml | 4 +- 6 files changed, 108 insertions(+), 44 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 32340563..f780e4d5 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -19,9 +19,12 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + ENV_FILE: envs/environment.yaml + jobs: - run: - name: Run + run-tests: + name: OS strategy: fail-fast: false matrix: @@ -29,14 +32,6 @@ jobs: - macos - windows - ubuntu - inhouse: - - stable-inhouse-deps - - dev-inhouse-deps - exclude: - - os: macos - inhouse: dev-inhouse-deps - - os: windows - inhouse: dev-inhouse-deps runs-on: ${{ matrix.os }}-latest defaults: @@ -46,25 +41,20 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Setup secrets + - name: Setup secrets & cache dates run: | echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc - - - name: Set cache dates - run: | echo "week=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_ENV # env - - name: Cache data and cutouts folders - uses: actions/cache@v4 + - uses: actions/cache@v4 with: path: | data cutouts key: data-cutouts-${{ env.week }} - - name: Setup Conda - uses: conda-incubator/setup-miniconda@v3 + - uses: conda-incubator/setup-miniconda@v3 with: activate-environment: pypsa-eur @@ -72,31 +62,105 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.CONDA }}/envs - key: conda-${{ runner.os }}--${{ runner.arch }}--${{ env.today }}-${{ hashFiles('envs/environment.yaml') }} + key: conda-${{ runner.os }}-${{ runner.arch }}-${{ env.today }}-${hashFiles('${{ env.ENV_FILE }}')} id: cache-env - name: Update environment if: steps.cache-env.outputs.cache-hit != 'true' - run: conda env update -n pypsa-eur -f envs/environment.yaml - - - name: Install inhouse packages from master - if: matrix.inhouse == 'dev-inhouse-deps' - run: | - python -m pip install uv - uv pip install git+https://github.com/PyPSA/pypsa.git@master - uv pip install git+https://github.com/PyPSA/atlite.git@master - uv pip install git+https://github.com/PyPSA/powerplantmatching.git@master - uv pip install git+https://github.com/PyPSA/linopy.git@master + run: conda env update -n pypsa-eur -f ${{ env.ENV_FILE }} - name: Run snakemake test workflows run: | + conda activate pypsa-eur make test - name: Upload artifacts - if: matrix.os == 'ubuntu' && matrix.inhouse == 'stable-inhouse-deps' - uses: actions/upload-artifact@v4.4.0 + uses: actions/upload-artifact@v3.2.1-node20 with: - name: resources-results + name: logs-${{ matrix.inhouse }} path: | - results - retention-days: 7 + logs + .snakemake/log + retention-days: 3 + + run-tests-on-dev-deps: + name: Inhouse + strategy: + fail-fast: false + matrix: + inhouse: + - pypsa + - atlite + - powerplantmatching + - linopy + runs-on: ubuntu-latest + + defaults: + run: + shell: bash -l {0} + + steps: + - uses: actions/checkout@v4 + + # Only run checks if package is not pinned + - name: Check if inhouse package is pinned + run: | + grep_line=$(grep -- '- ${{ matrix.inhouse }}' ${{ env.ENV_FILE }}) + if [[ $grep_line == *"<"* || $grep_line == *"=="* ]]; then + echo "pinned=true" >> $GITHUB_ENV + else + echo "pinned=false" >> $GITHUB_ENV + fi + + - name: Setup secrets & cache dates + if: env.pinned == 'false' + run: | + echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc + echo "week=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts + echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_ENV # env + + - uses: actions/cache@v4 + if: env.pinned == 'false' + with: + path: | + data + cutouts + key: data-cutouts-${{ env.week }} + + - uses: conda-incubator/setup-miniconda@v3 + if: env.pinned == 'false' + with: + activate-environment: pypsa-eur + + - name: Cache Conda env + if: env.pinned == 'false' + uses: actions/cache@v4 + with: + path: ${{ env.CONDA }}/envs + key: conda-${{ runner.os }}-${{ runner.arch }}-${{ matrix.inhouse }}-${{ env.today }}-${hashFiles('${{ env.ENV_FILE }}')} + id: cache-env + + - name: Update environment + if: env.pinned == 'false' && steps.cache-env.outputs.cache-hit != 'true' + run: conda env update -n pypsa-eur -f ${{ env.ENV_FILE }} + + - name: Install inhouse packages from master + if: env.pinned == 'false' + run: | + python -m pip install git+https://github.com/PyPSA/${{ matrix.inhouse }}.git@master + + - name: Run snakemake test workflows + if: env.pinned == 'false' + run: | + conda activate pypsa-eur + make test + + - name: Upload artifacts + if: env.pinned == 'false' + uses: actions/upload-artifact@v4 + with: + name: logs-${{ matrix.inhouse }} + path: | + logs + .snakemake/log + retention-days: 3 diff --git a/config/test/config.electricity.yaml b/config/test/config.electricity.yaml index 38fa31ab..147d735f 100644 --- a/config/test/config.electricity.yaml +++ b/config/test/config.electricity.yaml @@ -79,8 +79,8 @@ lines: solving: solver: - name: glpk - options: "glpk-default" + name: highs + options: highs-default plotting: diff --git a/config/test/config.myopic.yaml b/config/test/config.myopic.yaml index 0ede9aa7..ba4c7779 100644 --- a/config/test/config.myopic.yaml +++ b/config/test/config.myopic.yaml @@ -86,8 +86,8 @@ industry: solving: solver: - name: glpk - options: glpk-default + name: highs + options: highs-default mem: 4000 plotting: diff --git a/config/test/config.overnight.yaml b/config/test/config.overnight.yaml index 92379ae2..e14ff86b 100644 --- a/config/test/config.overnight.yaml +++ b/config/test/config.overnight.yaml @@ -80,8 +80,8 @@ industry: solving: solver: - name: glpk - options: glpk-default + name: highs + options: highs-default mem: 4000 plotting: diff --git a/config/test/config.perfect.yaml b/config/test/config.perfect.yaml index 781b3fd4..742b3012 100644 --- a/config/test/config.perfect.yaml +++ b/config/test/config.perfect.yaml @@ -85,8 +85,8 @@ industry: solving: solver: - name: glpk - options: glpk-default + name: highs + options: highs-default mem: 4000 plotting: diff --git a/config/test/config.scenarios.yaml b/config/test/config.scenarios.yaml index a9a826fd..d238703d 100644 --- a/config/test/config.scenarios.yaml +++ b/config/test/config.scenarios.yaml @@ -57,5 +57,5 @@ renewable: solving: solver: - name: glpk - options: "glpk-default" + name: highs + options: highs-default