From ec803a372ec3eb9220d1e91b32f7562d27c37414 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 24 Sep 2020 10:09:11 +0200 Subject: [PATCH 01/14] add cost uncertainty opts for suptechs (#189) --- scripts/prepare_network.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 1fcba99f..b38b958c 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -182,7 +182,8 @@ if __name__ == "__main__": for o in opts: oo = o.split("+") - if oo[0].startswith(tuple(n.carriers.index)): + suptechs = map(lambda c: c.split("-", 2)[0], n.carriers.index) + if oo[0].startswith(tuple(suptechs)): carrier = oo[0] cost_factor = float(oo[1]) if carrier == "AC": # lines do not have carrier From 047276b2dc13a1a296ee9f34c20562bfc78ecc16 Mon Sep 17 00:00:00 2001 From: Martha Maria <53824825+eb5194@users.noreply.github.com> Date: Thu, 24 Sep 2020 10:13:13 +0200 Subject: [PATCH 02/14] Fix clustering sites (#191) * fix clustering of offwind-ac and offwind-dc in sites option * add release nodes Co-authored-by: martha.frysztacki --- doc/release_notes.rst | 2 ++ scripts/cluster_network.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index f5e3c377..b33a6b6a 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -25,6 +25,8 @@ Upcoming Release * Don't remove capital costs from lines and links, when imposing a line volume limit (wildcard ``lv``) or a line cost limit (``lc``). Previously, these were removed to move the expansion in direction of the limit. +* Fix bug of clustering offwind-{ac,dc} sites in the option of high-resolution sites for renewables. Now, there are more sites for offwind-{ac,dc} available than network nodes. Before, they were clustered to the resolution of the network. (e.g. elec_s1024_37m.nc: 37 network nodes, 1024 sites) + PyPSA-Eur 0.2.0 (8th June 2020) ================================== diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index ff85bcf3..6a8e526b 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -333,7 +333,7 @@ if __name__ == "__main__": renewable_carriers = pd.Index([tech for tech in n.generators.carrier.unique() - if tech.split('-', 2)[0] in snakemake.config['renewable']]) + if tech in snakemake.config['renewable']]) if snakemake.wildcards.clusters.endswith('m'): n_clusters = int(snakemake.wildcards.clusters[:-1]) From 9331f1ec46ceafd8aeffbbb655e3ec9acee3acdb Mon Sep 17 00:00:00 2001 From: "martha.frysztacki" Date: Thu, 24 Sep 2020 17:46:20 +0200 Subject: [PATCH 03/14] cluster_network: The 'squeeze' parameter is deprecated and will be removed in a future release; works just fine without it --- scripts/cluster_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 6a8e526b..ce1e4319 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -257,7 +257,7 @@ def busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights=None, algori else: raise ValueError(f"`algorithm` must be one of 'kmeans', 'spectral' or 'louvain'. Is {algorithm}.") - return (n.buses.groupby(['country', 'sub_network'], group_keys=False, squeeze=True) + return (n.buses.groupby(['country', 'sub_network'], group_keys=False) .apply(busmap_for_country).rename('busmap')) def plot_busmap_for_n_clusters(n, n_clusters=50): From c543fb39c6201b153ec869d8018f70cddcb9a97e Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 25 Sep 2020 12:34:34 +0200 Subject: [PATCH 04/14] cluster: fix removing deprec squeeze=True in groupby (#194) --- scripts/cluster_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index ce1e4319..a4da7cda 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -258,7 +258,7 @@ def busmap_for_n_clusters(n, n_clusters, solver_name, focus_weights=None, algori raise ValueError(f"`algorithm` must be one of 'kmeans', 'spectral' or 'louvain'. Is {algorithm}.") return (n.buses.groupby(['country', 'sub_network'], group_keys=False) - .apply(busmap_for_country).rename('busmap')) + .apply(busmap_for_country).squeeze().rename('busmap')) def plot_busmap_for_n_clusters(n, n_clusters=50): busmap = busmap_for_n_clusters(n, n_clusters) From fcb895a098cb852b5b5f9e15744e97e436adf29c Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 25 Sep 2020 14:01:00 +0200 Subject: [PATCH 05/14] environment: limit snakemake to 5.24.2 (#195) until https://github.com/snakemake/snakemake/issues/635 resolved --- environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yaml b/environment.yaml index ce49420f..ca8b0862 100644 --- a/environment.yaml +++ b/environment.yaml @@ -20,7 +20,7 @@ dependencies: - scikit-learn - pycountry - seaborn - - snakemake-minimal + - snakemake-minimal<=5.24.2 # until https://github.com/snakemake/snakemake/issues/635 closed - memory_profiler - yaml - pytables From 456daa626066a39981985164fe641721a1171870 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 25 Sep 2020 14:08:46 +0200 Subject: [PATCH 06/14] travis: use mamba for faster builds (#196) * travis: use mamba for faster builds * doc: add release notes [skip travis] --- .travis.yml | 7 ++++++- doc/release_notes.rst | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a8e6e36e..1cebdf8a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,10 +16,15 @@ before_install: # install conda environment - conda env create -f ./environment.yaml + - conda install -c conda-forge mamba + - mamba env create -f ./envs/environment.yaml - conda activate pypsa-eur # install open-source solver - - conda install -c conda-forge ipopt glpk + - mamba install -c conda-forge ipopt glpk + + # list packages for easier debugging + - conda list script: - cp ./test/config.test1.yaml ./config.yaml diff --git a/doc/release_notes.rst b/doc/release_notes.rst index b33a6b6a..12d940d3 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -27,6 +27,8 @@ Upcoming Release * Fix bug of clustering offwind-{ac,dc} sites in the option of high-resolution sites for renewables. Now, there are more sites for offwind-{ac,dc} available than network nodes. Before, they were clustered to the resolution of the network. (e.g. elec_s1024_37m.nc: 37 network nodes, 1024 sites) +* Use `mamba` (https://github.com/mamba-org/mamba) for faster Travis CI builds (`#196 `_) + PyPSA-Eur 0.2.0 (8th June 2020) ================================== From baff29572199f9685c66b4dc900f0a8b9df204d8 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 25 Sep 2020 14:13:55 +0200 Subject: [PATCH 07/14] travis: remove unnecessary env create via conda --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1cebdf8a..011753f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,6 @@ before_install: - source conda4travis.sh # install conda environment - - conda env create -f ./environment.yaml - conda install -c conda-forge mamba - mamba env create -f ./envs/environment.yaml - conda activate pypsa-eur From c7dd302262b4b51cc458c94403b5f8dabf3b731e Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 25 Sep 2020 14:16:04 +0200 Subject: [PATCH 08/14] travis: fix path to environment file --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 011753f4..df8d1406 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ before_install: # install conda environment - conda install -c conda-forge mamba - - mamba env create -f ./envs/environment.yaml + - mamba env create -f ./environment.yaml - conda activate pypsa-eur # install open-source solver From 336392fcf8d07c2ee997c986870fbee2e0fe85fb Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 25 Sep 2020 14:44:31 +0200 Subject: [PATCH 09/14] travis: debug which solver causes problem --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index df8d1406..ba00fe2a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,8 @@ before_install: - conda activate pypsa-eur # install open-source solver - - mamba install -c conda-forge ipopt glpk + - mamba install -c conda-forge ipopt + - mamba install -c conda-forge glpk # list packages for easier debugging - conda list From fc7f04d1d87396eb7d639132e6401859f2767974 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 25 Sep 2020 14:53:17 +0200 Subject: [PATCH 10/14] travis: switch solver installation back to conda --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ba00fe2a..f08cd97e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,8 +20,8 @@ before_install: - conda activate pypsa-eur # install open-source solver - - mamba install -c conda-forge ipopt - mamba install -c conda-forge glpk + - conda install -c conda-forge ipopt # list packages for easier debugging - conda list From 06b8fe1e9862c5e8b5941d543b1158e54c514789 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 25 Sep 2020 15:06:39 +0200 Subject: [PATCH 11/14] environment: add mamba --- environment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yaml b/environment.yaml index ca8b0862..d39fb386 100644 --- a/environment.yaml +++ b/environment.yaml @@ -11,6 +11,7 @@ channels: dependencies: - python - pip + - mamba - pypsa>=0.17.1 - atlite=0.0.3 From db3a0e8e5c490e95c0fdc2442c8a19cb52cf073b Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 25 Sep 2020 15:18:59 +0200 Subject: [PATCH 12/14] travis: all via mamba --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f08cd97e..17bdb73a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,8 +20,7 @@ before_install: - conda activate pypsa-eur # install open-source solver - - mamba install -c conda-forge glpk - - conda install -c conda-forge ipopt + - mamba install -c conda-forge glpk ipopt # list packages for easier debugging - conda list From b81dfcaadaa2e7fa3f58cadd1ee6b2079e8e75ce Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 25 Sep 2020 15:34:35 +0200 Subject: [PATCH 13/14] environment: limit libgdal to 3.0.2 --- environment.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/environment.yaml b/environment.yaml index d39fb386..ca8b0862 100644 --- a/environment.yaml +++ b/environment.yaml @@ -11,7 +11,6 @@ channels: dependencies: - python - pip - - mamba - pypsa>=0.17.1 - atlite=0.0.3 From 9966038837232b05e1ad09a97d3d251eef43fcda Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 25 Sep 2020 15:47:44 +0200 Subject: [PATCH 14/14] environment: limit libgdal to 3.0.2 (2nd try) --- environment.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/environment.yaml b/environment.yaml index ca8b0862..59c0cbbc 100644 --- a/environment.yaml +++ b/environment.yaml @@ -11,6 +11,7 @@ channels: dependencies: - python - pip + - mamba # esp for windows build - pypsa>=0.17.1 - atlite=0.0.3 @@ -48,7 +49,7 @@ dependencies: - geopandas - rasterio - shapely - - libgdal + - libgdal<=3.0.2 # Solvers - gurobi:gurobi # until https://github.com/conda-forge/pypsa-feedstock/issues/4 closed