From 4349903a72c7dcf5848f9b8c7d6b68a8c1292dc0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 30 Dec 2023 14:36:18 +0100 Subject: [PATCH 01/21] env: remove snakemake upper version constraint --- envs/environment.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 535acbdb..b9f9ac99 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -20,8 +20,7 @@ dependencies: - openpyxl!=3.1.1 - pycountry - seaborn - # snakemake 8 introduced a number of breaking changes which the workflow has yet to be made compatible with -- snakemake-minimal>=7.7.0,<8.0.0 +- snakemake-minimal>=7.7.0 - memory_profiler - yaml - pytables From 48850431fbc296d4178933ea0f869bc515068b92 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 30 Dec 2023 14:37:45 +0100 Subject: [PATCH 02/21] Snakefile: switch to new storage providers --- Snakefile | 4 ---- envs/environment.yaml | 1 + rules/build_sector.smk | 14 +++++-------- rules/retrieve.smk | 47 ++++++++++++++---------------------------- 4 files changed, 21 insertions(+), 45 deletions(-) diff --git a/Snakefile b/Snakefile index 83530df7..42988ebc 100644 --- a/Snakefile +++ b/Snakefile @@ -5,10 +5,6 @@ from os.path import normpath, exists from shutil import copyfile, move, rmtree -from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider - -HTTP = HTTPRemoteProvider() - from snakemake.utils import min_version min_version("7.7") diff --git a/envs/environment.yaml b/envs/environment.yaml index b9f9ac99..ce733ed9 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -59,3 +59,4 @@ dependencies: - pip: - tsam>=2.3.1 + - snakemake-storage-plugin-http \ No newline at end of file diff --git a/rules/build_sector.smk b/rules/build_sector.smk index dd49fc6f..3f71ba72 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -85,9 +85,8 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: rule build_gas_input_locations: input: - lng=HTTP.remote( + lng=storage( "https://globalenergymonitor.org/wp-content/uploads/2023/07/Europe-Gas-Tracker-2023-03-v3.xlsx", - keep_local=True, ), entry="data/gas_network/scigrid-gas/data/IGGIELGN_BorderPoints.geojson", production="data/gas_network/scigrid-gas/data/IGGIELGN_Productions.geojson", @@ -268,9 +267,8 @@ rule build_biomass_potentials: params: biomass=config["biomass"], input: - enspreso_biomass=HTTP.remote( + enspreso_biomass=storage( "https://zenodo.org/records/10356004/files/ENSPRESO_BIOMASS.xlsx", - keep_local=True, ), nuts2="data/bundle-sector/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21 regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -300,9 +298,8 @@ if config["sector"]["biomass_transport"] or config["sector"]["biomass_spatial"]: rule build_biomass_transport_costs: input: - transport_cost_data=HTTP.remote( - "publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass potentials in europe_web rev.pdf", - keep_local=True, + transport_cost_data=storage( + "https://publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass potentials in europe_web rev.pdf", ), output: biomass_transport_costs=RESOURCES + "biomass_transport_costs.csv", @@ -334,9 +331,8 @@ if config["sector"]["regional_co2_sequestration_potential"]["enable"]: "regional_co2_sequestration_potential" ], input: - sequestration_potential=HTTP.remote( + sequestration_potential=storage( "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson", - keep_local=True, ), regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 0483a833..a35cc53c 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -64,9 +64,8 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True rule retrieve_cutout: input: - HTTP.remote( - "zenodo.org/record/6382570/files/{cutout}.nc", - static=True, + storage( + "https://zenodo.org/record/6382570/files/{cutout}.nc", ), output: protected("cutouts/" + CDIR + "{cutout}.nc"), @@ -83,12 +82,11 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", T rule retrieve_cost_data: input: - HTTP.remote( - "raw.githubusercontent.com/PyPSA/technology-data/{}/outputs/".format( + storage( + "https://raw.githubusercontent.com/PyPSA/technology-data/{}/outputs/".format( config["costs"]["version"] ) + "costs_{year}.csv", - keep_local=True, ), output: "data/costs_{year}.csv", @@ -107,10 +105,8 @@ if config["enable"]["retrieve"] and config["enable"].get( rule retrieve_natura_raster: input: - HTTP.remote( - "zenodo.org/record/4706686/files/natura.tiff", - keep_local=True, - static=True, + storage( + "https://zenodo.org/record/4706686/files/natura.tiff", ), output: RESOURCES + "natura.tiff", @@ -188,14 +184,12 @@ if config["enable"]["retrieve"]: rule retrieve_electricity_demand: input: - HTTP.remote( - "data.open-power-system-data.org/time_series/{version}/time_series_60min_singleindex.csv".format( + storage( + "https://data.open-power-system-data.org/time_series/{version}/time_series_60min_singleindex.csv".format( version="2019-06-05" if config["snapshots"]["end"] < "2019" else "2020-10-06" ), - keep_local=True, - static=True, ), output: RESOURCES + "load_raw.csv", @@ -212,10 +206,8 @@ if config["enable"]["retrieve"]: rule retrieve_ship_raster: input: - HTTP.remote( + storage( "https://zenodo.org/record/6953563/files/shipdensity_global.zip", - keep_local=True, - static=True, ), output: protected("data/shipdensity_global.zip"), @@ -234,9 +226,8 @@ if config["enable"]["retrieve"]: # Website: https://land.copernicus.eu/global/products/lc rule download_copernicus_land_cover: input: - HTTP.remote( - "zenodo.org/record/3939050/files/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif", - static=True, + storage( + "https://zenodo.org/record/3939050/files/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif", ), output: "data/Copernicus_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif", @@ -279,11 +270,7 @@ if config["enable"]["retrieve"]: # Website: https://www.protectedplanet.net/en/thematic-areas/wdpa rule download_wdpa: input: - HTTP.remote( - url, - static=True, - keep_local=True, - ), + storage(url), params: zip="data/WDPA_shp.zip", folder=directory("data/WDPA"), @@ -305,10 +292,8 @@ if config["enable"]["retrieve"]: # extract the main zip and then merge the contained 3 zipped shapefiles # Website: https://www.protectedplanet.net/en/thematic-areas/marine-protected-areas input: - HTTP.remote( - f"d1gam3xoknrgr2.cloudfront.net/current/WDPA_WDOECM_{bYYYY}_Public_marine_shp.zip", - static=True, - keep_local=True, + storage( + f"https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_WDOECM_{bYYYY}_Public_marine_shp.zip", ), params: zip="data/WDPA_WDOECM_marine.zip", @@ -330,10 +315,8 @@ if config["enable"]["retrieve"]: rule retrieve_monthly_co2_prices: input: - HTTP.remote( + storage( "https://www.eex.com/fileadmin/EEX/Downloads/EUA_Emission_Spot_Primary_Market_Auction_Report/Archive_Reports/emission-spot-primary-market-auction-report-2019-data.xls", - keep_local=True, - static=True, ), output: "data/validation/emission-spot-primary-market-auction-report-2019-data.xls", From 98e47eab3cef408dca6a9fc19a6e972b280aaa9f Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sun, 31 Dec 2023 13:00:21 +0100 Subject: [PATCH 03/21] mock_snakemake: adjust to snakemake version 8 --- scripts/_helpers.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 398f3a30..c3066965 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -214,9 +214,17 @@ def mock_snakemake(rulename, root_dir=None, configfiles=[], **wildcards): import os import snakemake as sm - from packaging.version import Version, parse from pypsa.descriptors import Dict + from snakemake.api import Workflow + from snakemake.common import SNAKEFILE_CHOICES from snakemake.script import Snakemake + from snakemake.settings import ( + ConfigSettings, + DAGSettings, + ResourceSettings, + StorageSettings, + WorkflowSettings, + ) script_dir = Path(__file__).parent.resolve() if root_dir is None: @@ -233,17 +241,26 @@ def mock_snakemake(rulename, root_dir=None, configfiles=[], **wildcards): f" {root_dir} or scripts directory {script_dir}" ) try: - for p in sm.SNAKEFILE_CHOICES: + for p in SNAKEFILE_CHOICES: if os.path.exists(p): snakefile = p break - kwargs = ( - dict(rerun_triggers=[]) if parse(sm.__version__) > Version("7.7.0") else {} - ) if isinstance(configfiles, str): configfiles = [configfiles] - workflow = sm.Workflow(snakefile, overwrite_configfiles=configfiles, **kwargs) + resource_settings = ResourceSettings() + config_settings = ConfigSettings(configfiles=configfiles) + workflow_settings = WorkflowSettings() + storage_settings = StorageSettings() + dag_settings = DAGSettings(rerun_triggers=[]) + workflow = Workflow( + config_settings, + resource_settings, + workflow_settings, + storage_settings, + dag_settings, + storage_provider_settings=dict(), + ) workflow.include(snakefile) if configfiles: From ea95f2b38c15719944f9bf413ae8683702593c92 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sun, 31 Dec 2023 13:01:17 +0100 Subject: [PATCH 04/21] bumb minimum snakemake version to v8 --- Snakefile | 2 +- envs/environment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Snakefile b/Snakefile index 42988ebc..468d06f2 100644 --- a/Snakefile +++ b/Snakefile @@ -7,7 +7,7 @@ from shutil import copyfile, move, rmtree from snakemake.utils import min_version -min_version("7.7") +min_version("8") if not exists("config/config.yaml"): diff --git a/envs/environment.yaml b/envs/environment.yaml index ce733ed9..7da973c6 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -20,7 +20,7 @@ dependencies: - openpyxl!=3.1.1 - pycountry - seaborn -- snakemake-minimal>=7.7.0 +- snakemake-minimal>=8 - memory_profiler - yaml - pytables From 19a50fb8c5c72fab188e305a5a8159c129df9ddc Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sun, 31 Dec 2023 14:07:45 +0100 Subject: [PATCH 05/21] rules: add keep_local=True back in for storage providers --- rules/build_sector.smk | 4 ++++ rules/retrieve.smk | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 3f71ba72..0fea3c99 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -87,6 +87,7 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: input: lng=storage( "https://globalenergymonitor.org/wp-content/uploads/2023/07/Europe-Gas-Tracker-2023-03-v3.xlsx", + keep_local=True, ), entry="data/gas_network/scigrid-gas/data/IGGIELGN_BorderPoints.geojson", production="data/gas_network/scigrid-gas/data/IGGIELGN_Productions.geojson", @@ -269,6 +270,7 @@ rule build_biomass_potentials: input: enspreso_biomass=storage( "https://zenodo.org/records/10356004/files/ENSPRESO_BIOMASS.xlsx", + keep_local=True, ), nuts2="data/bundle-sector/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21 regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -300,6 +302,7 @@ if config["sector"]["biomass_transport"] or config["sector"]["biomass_spatial"]: input: transport_cost_data=storage( "https://publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass potentials in europe_web rev.pdf", + keep_local=True, ), output: biomass_transport_costs=RESOURCES + "biomass_transport_costs.csv", @@ -333,6 +336,7 @@ if config["sector"]["regional_co2_sequestration_potential"]["enable"]: input: sequestration_potential=storage( "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson", + keep_local=True, ), regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index a35cc53c..13120640 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -87,6 +87,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", T config["costs"]["version"] ) + "costs_{year}.csv", + keep_local=True, ), output: "data/costs_{year}.csv", @@ -107,6 +108,7 @@ if config["enable"]["retrieve"] and config["enable"].get( input: storage( "https://zenodo.org/record/4706686/files/natura.tiff", + keep_local=True, ), output: RESOURCES + "natura.tiff", @@ -190,6 +192,7 @@ if config["enable"]["retrieve"]: if config["snapshots"]["end"] < "2019" else "2020-10-06" ), + keep_local=True, ), output: RESOURCES + "load_raw.csv", @@ -208,6 +211,7 @@ if config["enable"]["retrieve"]: input: storage( "https://zenodo.org/record/6953563/files/shipdensity_global.zip", + keep_local=True, ), output: protected("data/shipdensity_global.zip"), @@ -270,7 +274,7 @@ if config["enable"]["retrieve"]: # Website: https://www.protectedplanet.net/en/thematic-areas/wdpa rule download_wdpa: input: - storage(url), + storage(url, keep_local=True), params: zip="data/WDPA_shp.zip", folder=directory("data/WDPA"), @@ -294,6 +298,7 @@ if config["enable"]["retrieve"]: input: storage( f"https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_WDOECM_{bYYYY}_Public_marine_shp.zip", + keep_local=True, ), params: zip="data/WDPA_WDOECM_marine.zip", @@ -317,6 +322,7 @@ if config["enable"]["retrieve"]: input: storage( "https://www.eex.com/fileadmin/EEX/Downloads/EUA_Emission_Spot_Primary_Market_Auction_Report/Archive_Reports/emission-spot-primary-market-auction-report-2019-data.xls", + keep_local=True, ), output: "data/validation/emission-spot-primary-market-auction-report-2019-data.xls", From 27ee2666be6300ac6a07f2e74c5ec6e1d49e2f64 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 31 Dec 2023 13:13:08 +0000 Subject: [PATCH 06/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- envs/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 7da973c6..36dbfcae 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -59,4 +59,4 @@ dependencies: - pip: - tsam>=2.3.1 - - snakemake-storage-plugin-http \ No newline at end of file + - snakemake-storage-plugin-http From 3d94e8161bada538b08d2fbed1de3ada01c00e58 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Wed, 28 Feb 2024 14:01:36 +0000 Subject: [PATCH 07/21] Adapt disabling transmission expansion to also handle cost limit --- scripts/add_brownfield.py | 66 ++++++++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 19 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 1e175d87..1bccf563 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -124,29 +124,57 @@ def add_brownfield(n, n_p, year): n.links.loc[new_pipes, "p_nom_min"] = 0.0 -def disable_grid_expansion_if_LV_limit_hit(n): - if "lv_limit" not in n.global_constraints.index: - return +def disable_grid_expansion_if_limit_hit(n): + """ + Check if transmission expansion limit is already reached; then turn off. - total_expansion = ( - n.lines.eval("s_nom_min * length").sum() - + n.links.query("carrier == 'DC'").eval("p_nom_min * length").sum() - ).sum() + In particular, this function checks if the total transmission + capital cost or volume implied by s_nom_min and p_nom_min are + numerically close to the respective global limit set in + n.global_constraints. If so, the nominal capacities are set to the + minimum and extendable is turned off; the corresponding global + constraint is then dropped. + """ + cols = {"cost": "capital_cost", "volume": "length"} + for limit_type in ["cost", "volume"]: + glcs = n.global_constraints.query( + f"type == 'transmission_expansion_{limit_type}_limit'" + ) - lv_limit = n.global_constraints.at["lv_limit", "constant"] + for name, glc in glcs.iterrows(): + total_expansion = ( + ( + n.lines.query("p_nom_extendable") + .eval(f"s_nom_min * {cols[limit_type]}") + .sum() + ) + + ( + n.links.query("carrier == 'DC' and p_nom_extendable") + .eval(f"p_nom_min * {cols[limit_type]}") + .sum() + ) + ).sum() - # allow small numerical differences - if lv_limit - total_expansion < 1: - logger.info("LV is already reached, disabling expansion and LV limit") - extendable_acs = n.lines.query("s_nom_extendable").index - n.lines.loc[extendable_acs, "s_nom_extendable"] = False - n.lines.loc[extendable_acs, "s_nom"] = n.lines.loc[extendable_acs, "s_nom_min"] + # Allow small numerical differences + if np.abs(glc.constant - total_expansion) / glc.constant < 1e-6: + logger.info( + f"Transmission expansion {limit_type} is already reached, disabling expansion and limit" + ) + extendable_acs = n.lines.query("s_nom_extendable").index + n.lines.loc[extendable_acs, "s_nom_extendable"] = False + n.lines.loc[extendable_acs, "s_nom"] = n.lines.loc[ + extendable_acs, "s_nom_min" + ] - extendable_dcs = n.links.query("carrier == 'DC' and p_nom_extendable").index - n.links.loc[extendable_dcs, "p_nom_extendable"] = False - n.links.loc[extendable_dcs, "p_nom"] = n.links.loc[extendable_dcs, "p_nom_min"] + extendable_dcs = n.links.query( + "carrier == 'DC' and p_nom_extendable" + ).index + n.links.loc[extendable_dcs, "p_nom_extendable"] = False + n.links.loc[extendable_dcs, "p_nom"] = n.links.loc[ + extendable_dcs, "p_nom_min" + ] - n.global_constraints.drop("lv_limit", inplace=True) + n.global_constraints.drop(name, inplace=True) def adjust_renewable_profiles(n, input_profiles, params, year): @@ -233,7 +261,7 @@ if __name__ == "__main__": add_brownfield(n, n_p, year) - disable_grid_expansion_if_LV_limit_hit(n) + disable_grid_expansion_if_limit_hit(n) n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) n.export_to_netcdf(snakemake.output[0]) From 61e4123176cc97cb7d9942e7e9e4ede04d13a42d Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Wed, 28 Feb 2024 15:11:32 +0100 Subject: [PATCH 08/21] Add release note --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index d6072d7d..612cb1a5 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -81,6 +81,8 @@ Upcoming Release * Bugfix: allow modelling sector-coupled landlocked regions. (Fixed handling of offshore wind.) +* Adapt the disabling of transmission expansion in myopic foresight optimisations when limit is already reached to also handle cost limits. + PyPSA-Eur 0.10.0 (19th February 2024) ===================================== From 09502cfbfa6034023505403449ff9db8135a7de7 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Mar 2024 11:26:00 +0100 Subject: [PATCH 09/21] add memory logger to solve_sector_network_myopic rule --- rules/solve_myopic.smk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 5eb30233..a081fbe6 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -126,6 +126,8 @@ rule solve_sector_network_myopic: log: solver=RESULTS + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log", + memory=RESULTS + + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_memory.log", python=RESULTS + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log", threads: solver_threads From b6542d275f20696a3d223b24d3ee76a50e734ae2 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Mar 2024 17:45:54 +0100 Subject: [PATCH 10/21] add snakemake slurm plugin to env --- envs/environment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/envs/environment.yaml b/envs/environment.yaml index 36dbfcae..3b85f7a9 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -60,3 +60,4 @@ dependencies: - pip: - tsam>=2.3.1 - snakemake-storage-plugin-http + - snakemake-executor-plugin-slurm From 105dc6470aeb48bf64fbdc2f0bc3c89dcc61bca5 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Mar 2024 17:51:02 +0100 Subject: [PATCH 11/21] replace remaining instances of HTTP.remote --- rules/build_sector.smk | 4 ++-- rules/retrieve.smk | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 36e20590..13565111 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -325,7 +325,7 @@ rule build_biomass_potentials: rule build_biomass_transport_costs: input: - transport_cost_data=HTTP.remote( + transport_cost_data=storage( "publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass potentials in europe_web rev.pdf", keep_local=True, ), @@ -350,7 +350,7 @@ rule build_sequestration_potentials: "sector", "regional_co2_sequestration_potential" ), input: - sequestration_potential=HTTP.remote( + sequestration_potential=storage( "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson", keep_local=True, ), diff --git a/rules/retrieve.smk b/rules/retrieve.smk index beae4f9c..6a54b4d1 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -236,7 +236,7 @@ if config["enable"]["retrieve"]: # Website: https://ec.europa.eu/jrc/en/luisa rule retrieve_luisa_land_cover: input: - HTTP.remote( + storage( "jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/LUISA/EUROPE/Basemaps/LandUse/2018/LATEST/LUISA_basemap_020321_50m.tif", static=True, ), From ab14c31f551fa6f461b9fc0b792eb4c606688562 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Mar 2024 19:15:51 +0100 Subject: [PATCH 12/21] build_electricity_demand adjustment to reduced function arguments --- scripts/build_electricity_demand.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py index 47befb8a..51dc6efd 100755 --- a/scripts/build_electricity_demand.py +++ b/scripts/build_electricity_demand.py @@ -273,7 +273,7 @@ if __name__ == "__main__": if "UA" in countries: # attach load of UA (best data only for entsoe transparency) - load_ua = load_timeseries(snakemake.input[0], "2018", ["UA"], False) + load_ua = load_timeseries(snakemake.input[0], "2018", ["UA"]) snapshot_year = str(snapshots.year.unique().item()) time_diff = pd.Timestamp("2018") - pd.Timestamp(snapshot_year) load_ua.index -= ( From d7ebeb384338a79b8e191dd5c4df62f33de60902 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Mar 2024 19:16:30 +0100 Subject: [PATCH 13/21] custom_powerplants: correct coordinates of some UA powerplants --- data/custom_powerplants.csv | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/data/custom_powerplants.csv b/data/custom_powerplants.csv index 4fd47498..84553a74 100644 --- a/data/custom_powerplants.csv +++ b/data/custom_powerplants.csv @@ -2,36 +2,36 @@ 1266,Khmelnitskiy,Nuclear,,PP,UA,1901.8916595755832,,0.0,0.0,0.0,0.0,1988.0,2005.0,,,50.3023,26.6466,[nan],"{'GEO': ['GEO3842'], 'GPD': ['WRI1005111'], 'CARMA': ['CARMA22000']}" 1268,Kaniv,Hydro,Reservoir,PP,UA,452.1656050955414,,0.0,0.0,0.0,0.0,1972.0,2003.0,,,49.76653,31.47165,[nan],"{'GEO': ['GEO43017'], 'GPD': ['WRI1005122'], 'CARMA': ['CARMA21140']}" 1269,Kahovska kakhovka,Hydro,Reservoir,PP,UA,352.45222929936307,,0.0,0.0,0.0,0.0,1955.0,1956.0,,,46.77858,33.36965,[nan],"{'GEO': ['GEO43018'], 'GPD': ['WRI1005118'], 'CARMA': ['CARMA20855']}" -1347,Kharkiv,Natural Gas,Steam Turbine,CHP,UA,494.94274967602314,,0.0,0.0,0.0,0.0,1979.0,1980.0,,,49.9719,36107,[nan],"{'GEO': ['GEO43027'], 'GPD': ['WRI1005126'], 'CARMA': ['CARMA21972']}" +1347,Kharkiv,Natural Gas,Steam Turbine,CHP,UA,494.94274967602314,,0.0,0.0,0.0,0.0,1979.0,1980.0,,,49.9719,36.107,[nan],"{'GEO': ['GEO43027'], 'GPD': ['WRI1005126'], 'CARMA': ['CARMA21972']}" 1348,Kremenchuk,Hydro,Reservoir,PP,UA,617.0382165605096,,0.0,0.0,0.0,0.0,1959.0,1960.0,,,49.07759,33.2505,[nan],"{'GEO': ['GEO43019'], 'GPD': ['WRI1005121'], 'CARMA': ['CARMA23072']}" 1377,Krivorozhskaya,Hard Coal,Steam Turbine,PP,UA,2600.0164509342876,,0.0,0.0,0.0,0.0,1965.0,1992.0,,,47.5432,33.6583,[nan],"{'GEO': ['GEO42989'], 'GPD': ['WRI1005100'], 'CARMA': ['CARMA23176']}" 1407,Zmiyevskaya zmiivskaya,Hard Coal,Steam Turbine,PP,UA,2028.3816283884514,,0.0,0.0,0.0,0.0,1960.0,2005.0,,,49.5852,36.5231,[nan],"{'GEO': ['GEO42999'], 'GPD': ['WRI1005103'], 'CARMA': ['CARMA51042']}" 1408,Pridneprovskaya,Hard Coal,Steam Turbine,CHP,UA,1627.3152609570984,,0.0,0.0,0.0,0.0,1959.0,1966.0,,,48.4051,35.1131,[nan],"{'GEO': ['GEO42990'], 'GPD': ['WRI1005102'], 'CARMA': ['CARMA35874']}" 1409,Kurakhovskaya,Hard Coal,Steam Turbine,PP,UA,1371.0015824607397,,0.0,0.0,0.0,0.0,1972.0,2003.0,,,47.9944,37.24022,[nan],"{'GEO': ['GEO42994'], 'GPD': ['WRI1005104'], 'CARMA': ['CARMA23339']}" -1410,Dobrotvorsky,Hard Coal,Steam Turbine,PP,UA,553.1949895604868,,0.0,0.0,0.0,0.0,1960.0,1964.0,,,50.2133,24375,[nan],"{'GEO': ['GEO42992'], 'GPD': ['WRI1005096'], 'CARMA': ['CARMA10971']}" +1410,Dobrotvorsky,Hard Coal,Steam Turbine,PP,UA,553.1949895604868,,0.0,0.0,0.0,0.0,1960.0,1964.0,,,50.2133,24.375,[nan],"{'GEO': ['GEO42992'], 'GPD': ['WRI1005096'], 'CARMA': ['CARMA10971']}" 1422,Zuyevskaya,Hard Coal,Steam Turbine,PP,UA,1147.87960333801,,0.0,0.0,0.0,0.0,1982.0,2007.0,,,48.0331,38.28615,[nan],"{'GEO': ['GEO42995'], 'GPD': ['WRI1005106'], 'CARMA': ['CARMA51083']}" 1423,Zaporozhye,Nuclear,,PP,UA,5705.67497872675,,0.0,0.0,0.0,0.0,1985.0,1996.0,,,47.5119,34.5863,[nan],"{'GEO': ['GEO6207'], 'GPD': ['WRI1005114'], 'CARMA': ['CARMA50875']}" 1424,Trypilska,Hard Coal,Steam Turbine,PP,UA,1659.5849686814602,,0.0,0.0,0.0,0.0,1969.0,1972.0,,,50.1344,30.7468,[nan],"{'GEO': ['GEO43000'], 'GPD': ['WRI1005099'], 'CARMA': ['CARMA46410']}" 1425,Tashlyk,Hydro,Pumped Storage,Store,UA,285.55968954109585,,0.0,0.0,0.0,0.0,2006.0,2007.0,,,47.7968,31.1811,[nan],"{'GEO': ['GEO43025'], 'GPD': ['WRI1005117'], 'CARMA': ['CARMA44696']}" 1426,Starobeshivska,Hard Coal,Steam Turbine,PP,UA,1636.5351774497733,,0.0,0.0,0.0,0.0,1961.0,1967.0,,,47.7997,38.00612,[nan],"{'GEO': ['GEO43003'], 'GPD': ['WRI1005105'], 'CARMA': ['CARMA43083']}" -1427,South,Nuclear,,PP,UA,2852.837489363375,,0.0,0.0,0.0,0.0,1983.0,1989.0,,,47812,31.22,[nan],"{'GEO': ['GEO5475'], 'GPD': ['WRI1005113'], 'CARMA': ['CARMA42555']}" +1427,South,Nuclear,,PP,UA,2852.837489363375,,0.0,0.0,0.0,0.0,1983.0,1989.0,,,47.812,31.22,[nan],"{'GEO': ['GEO5475'], 'GPD': ['WRI1005113'], 'CARMA': ['CARMA42555']}" 1428,Rovno rivne,Nuclear,,PP,UA,2695.931427448389,,0.0,0.0,0.0,0.0,1981.0,2006.0,,,51.3245,25.89744,[nan],"{'GEO': ['GEO5174'], 'GPD': ['WRI1005112'], 'CARMA': ['CARMA38114']}" -1429,Ladyzhinska,Hard Coal,Steam Turbine,PP,UA,1659.5849686814602,,0.0,0.0,0.0,0.0,1970.0,1971.0,,,48706,29.2202,[nan],"{'GEO': ['GEO42993'], 'GPD': ['WRI1005098'], 'CARMA': ['CARMA24024']}" -1430,Kiev,Hydro,Pumped Storage,PP,UA,635.8694635681177,,0.0,0.0,0.0,0.0,1964.0,1972.0,,,50.5998,30501,"[nan, nan]","{'GEO': ['GEO43024', 'GEO43023'], 'GPD': ['WRI1005123', 'WRI1005124'], 'CARMA': ['CARMA23516', 'CARMA23517']}" +1429,Ladyzhinska,Hard Coal,Steam Turbine,PP,UA,1659.5849686814602,,0.0,0.0,0.0,0.0,1970.0,1971.0,,,48.706,29.2202,[nan],"{'GEO': ['GEO42993'], 'GPD': ['WRI1005098'], 'CARMA': ['CARMA24024']}" +1430,Kiev,Hydro,Pumped Storage,PP,UA,635.8694635681177,,0.0,0.0,0.0,0.0,1964.0,1972.0,,,50.5998,30.501,"[nan, nan]","{'GEO': ['GEO43024', 'GEO43023'], 'GPD': ['WRI1005123', 'WRI1005124'], 'CARMA': ['CARMA23516', 'CARMA23517']}" 2450,Cet chisinau,Natural Gas,,PP,MD,306.0,,0.0,0.0,0.0,0.0,,,,,47.027550000000005,28.8801,"[nan, nan]","{'GPD': ['WRI1002985', 'WRI1002984'], 'CARMA': ['CARMA8450', 'CARMA8451']}" 2460,Hydropower che costesti,Hydro,,PP,MD,16.0,,0.0,0.0,0.0,0.0,1978.0,,,,47.8381,27.2246,[nan],"{'GPD': ['WRI1002987'], 'CARMA': ['CARMA9496']}" 2465,Moldavskaya gres,Hard Coal,,PP,MD,2520.0,,0.0,0.0,0.0,0.0,,,,,46.6292,29.9407,[nan],"{'GPD': ['WRI1002989'], 'CARMA': ['CARMA28979']}" -2466,Hydropower dubasari,Hydro,,PP,MD,48.0,,0.0,0.0,0.0,0.0,,,,,47.2778,29123,[nan],"{'GPD': ['WRI1002988'], 'CARMA': ['CARMA11384']}" +2466,Hydropower dubasari,Hydro,,PP,MD,48.0,,0.0,0.0,0.0,0.0,,,,,47.2778,29.123,[nan],"{'GPD': ['WRI1002988'], 'CARMA': ['CARMA11384']}" 2676,Cet nord balti,Natural Gas,,PP,MD,24.0,,0.0,0.0,0.0,0.0,,,,,47.7492,27.8938,[nan],"{'GPD': ['WRI1002986'], 'CARMA': ['CARMA3071']}" 2699,Dniprodzerzhynsk,Hydro,Reservoir,PP,UA,360.3503184713376,,0.0,0.0,0.0,0.0,1963.0,1964.0,,,48.5485,34.541015,[nan],"{'GEO': ['GEO43020'], 'GPD': ['WRI1005119']}" 2707,Burshtynska tes,Hard Coal,Steam Turbine,PP,UA,2212.779958241947,,0.0,0.0,0.0,0.0,1965.0,1984.0,,,49.21038,24.66654,[nan],"{'GEO': ['GEO42991'], 'GPD': ['WRI1005097']}" 2708,Danipro dnieper,Hydro,Reservoir,PP,UA,1484.8407643312103,,0.0,0.0,0.0,0.0,1932.0,1947.0,,,47.86944,35.08611,[nan],"{'GEO': ['GEO43016'], 'GPD': ['WRI1005120']}" 2709,Dniester,Hydro,Pumped Storage,Store,UA,612.7241020616891,,0.0,0.0,0.0,0.0,2009.0,2011.0,,,48.51361,27.47333,[nan],"{'GEO': ['GEO43022'], 'GPD': ['WRI1005116', 'WRI1005115']}" -2710,Kiev,Natural Gas,Steam Turbine,CHP,UA,458.2803237740955,,0.0,0.0,0.0,0.0,1982.0,1984.0,,,50532,30.6625,[nan],"{'GEO': ['GEO42998'], 'GPD': ['WRI1005125']}" +2710,Kiev,Natural Gas,Steam Turbine,CHP,UA,458.2803237740955,,0.0,0.0,0.0,0.0,1982.0,1984.0,,,50.532,30.6625,[nan],"{'GEO': ['GEO42998'], 'GPD': ['WRI1005125']}" 2712,Luganskaya,Hard Coal,Steam Turbine,PP,UA,1060.2903966575996,,0.0,0.0,0.0,0.0,1962.0,1969.0,,,48.74781,39.2624,[nan],"{'GEO': ['GEO42996'], 'GPD': ['WRI1005110']}" -2713,Slavyanskaya,Hard Coal,Steam Turbine,PP,UA,737.5933194139823,,0.0,0.0,0.0,0.0,1971.0,1971.0,,,48872,37.76567,[nan],"{'GEO': ['GEO43002'], 'GPD': ['WRI1005109']}" +2713,Slavyanskaya,Hard Coal,Steam Turbine,PP,UA,737.5933194139823,,0.0,0.0,0.0,0.0,1971.0,1971.0,,,48.872,37.76567,[nan],"{'GEO': ['GEO43002'], 'GPD': ['WRI1005109']}" 2714,Vuhlehirska uglegorskaya,Hard Coal,Steam Turbine,PP,UA,3319.1699373629203,,0.0,0.0,0.0,0.0,1972.0,1977.0,,,48.4633,38.20328,[nan],"{'GEO': ['GEO43001'], 'GPD': ['WRI1005107']}" 2715,Zaporiska,Hard Coal,Steam Turbine,PP,UA,3319.1699373629203,,0.0,0.0,0.0,0.0,1972.0,1977.0,,,47.5089,34.6253,[nan],"{'GEO': ['GEO42988'], 'GPD': ['WRI1005101']}" 3678,Mironovskaya,Hard Coal,,PP,UA,815.0,,0.0,0.0,0.0,0.0,,,,,48.3407,38.4049,[nan],"{'GPD': ['WRI1005108'], 'CARMA': ['CARMA28679']}" 3679,Kramatorskaya,Hard Coal,,PP,UA,120.0,,0.0,0.0,0.0,0.0,1974.0,,,,48.7477,37.5723,[nan],"{'GPD': ['WRI1075856'], 'CARMA': ['CARMA54560']}" -3680,Chernihiv,Hard Coal,,PP,UA,200.0,,0.0,0.0,0.0,0.0,1968.0,,,,51455,31.2602,[nan],"{'GPD': ['WRI1075853'], 'CARMA': ['CARMA8190']}" +3680,Chernihiv,Hard Coal,,PP,UA,200.0,,0.0,0.0,0.0,0.0,1968.0,,,,51.455,31.2602,[nan],"{'GPD': ['WRI1075853'], 'CARMA': ['CARMA8190']}" From 008539d635c1da9fff3d4852bd0ac0444e9cd52c Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 8 Mar 2024 19:29:25 +0100 Subject: [PATCH 14/21] add_electricity: fill max_hours also for nan values --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index a799caec..06f6348d 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -586,7 +586,7 @@ def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **par # fill missing max hours to params value and # assume no natural inflow due to lack of data max_hours = params.get("PHS_max_hours", 6) - phs = phs.replace({"max_hours": {0: max_hours}}) + phs = phs.replace({"max_hours": {0: max_hours, np.nan: max_hours}}) n.madd( "StorageUnit", phs.index, From c8eb46a39c7bced4c8038c627ff373c3cbe2ea4f Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 9 Mar 2024 12:07:15 +0100 Subject: [PATCH 15/21] add generic cluster snakemake plugin --- envs/environment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/envs/environment.yaml b/envs/environment.yaml index 316a662c..a5323923 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -62,4 +62,5 @@ dependencies: - tsam>=2.3.1 - snakemake-storage-plugin-http - snakemake-executor-plugin-slurm + - snakemake-executor-plugin-cluster-generic - highspy From 45e955a0d894f8885c73a5c8cb5664f3ea90b3f3 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 9 Mar 2024 12:07:47 +0100 Subject: [PATCH 16/21] add https:// prefix for all storage() calls --- rules/build_sector.smk | 2 +- rules/retrieve.smk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 13565111..8d41e893 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -326,7 +326,7 @@ rule build_biomass_potentials: rule build_biomass_transport_costs: input: transport_cost_data=storage( - "publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass potentials in europe_web rev.pdf", + "https://publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass potentials in europe_web rev.pdf", keep_local=True, ), output: diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 6a54b4d1..4cffc314 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -237,7 +237,7 @@ if config["enable"]["retrieve"]: rule retrieve_luisa_land_cover: input: storage( - "jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/LUISA/EUROPE/Basemaps/LandUse/2018/LATEST/LUISA_basemap_020321_50m.tif", + "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/LUISA/EUROPE/Basemaps/LandUse/2018/LATEST/LUISA_basemap_020321_50m.tif", static=True, ), output: From bcd02d94d613b1779cc900d24ded9b590e5f29d6 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 9 Mar 2024 12:10:45 +0100 Subject: [PATCH 17/21] retrieve: remove storage(static=True) call --- rules/retrieve.smk | 1 - 1 file changed, 1 deletion(-) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 4cffc314..c452969d 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -238,7 +238,6 @@ if config["enable"]["retrieve"]: input: storage( "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/LUISA/EUROPE/Basemaps/LandUse/2018/LATEST/LUISA_basemap_020321_50m.tif", - static=True, ), output: "data/LUISA_basemap_020321_50m.tif", From ed71978b92554af6bd240cbc072e2dfabd9e0ad8 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 9 Mar 2024 14:47:01 +0100 Subject: [PATCH 18/21] use humanfriendly walltime specifications --- Snakefile | 2 +- config/config.default.yaml | 4 ++-- envs/environment.yaml | 2 +- rules/solve_electricity.smk | 4 ++-- rules/solve_myopic.smk | 4 ++-- rules/solve_overnight.smk | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Snakefile b/Snakefile index c2d71cfd..58cb6677 100644 --- a/Snakefile +++ b/Snakefile @@ -9,7 +9,7 @@ import yaml from snakemake.utils import min_version -min_version("8") +min_version("8.5") from scripts._helpers import path_provider diff --git a/config/config.default.yaml b/config/config.default.yaml index e7bfe18f..d9f69206 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -813,8 +813,8 @@ solving: cbc-default: {} # Used in CI glpk-default: {} # Used in CI - mem: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2 - walltime: "12:00:00" + mem_mb: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2 + runtime: 6h #runtime in humanfriendly style https://humanfriendly.readthedocs.io/en/latest/ # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#plotting plotting: diff --git a/envs/environment.yaml b/envs/environment.yaml index a5323923..ee1d1605 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -20,7 +20,7 @@ dependencies: - openpyxl!=3.1.1 - pycountry - seaborn -- snakemake-minimal>=8 +- snakemake-minimal>=8.5 - memory_profiler - yaml - pytables diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index d0cab34c..8481c75b 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -29,7 +29,7 @@ rule solve_network: threads: solver_threads resources: mem_mb=memory, - walltime=config_provider("solving", "walltime", default="12:00:00"), + runtime=config_provider("solving", "runtime", default="6h"), shadow: "minimal" conda: @@ -60,7 +60,7 @@ rule solve_operations_network: threads: 4 resources: mem_mb=(lambda w: 10000 + 372 * int(w.clusters)), - walltime=config_provider("solving", "walltime", default="12:00:00"), + runtime=config_provider("solving", "runtime", default="6h"), shadow: "minimal" conda: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index a081fbe6..57b8a9d3 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -132,8 +132,8 @@ rule solve_sector_network_myopic: + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log", threads: solver_threads resources: - mem_mb=config_provider("solving", "mem"), - walltime=config_provider("solving", "walltime", default="12:00:00"), + mem_mb=config_provider("solving", "mem_mb"), + runtime=config_provider("solving", "runtime", default="6h"), benchmark: ( RESULTS diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index b212d453..26e6e220 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -30,8 +30,8 @@ rule solve_sector_network: + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log", threads: solver_threads resources: - mem_mb=config_provider("solving", "mem"), - walltime=config_provider("solving", "walltime", default="12:00:00"), + mem_mb=config_provider("solving", "mem_mb"), + runtime=config_provider("solving", "runtime", default="6h"), benchmark: ( RESULTS From 887aa532901027a02adccffa08367d331955399b Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 9 Mar 2024 15:29:39 +0100 Subject: [PATCH 19/21] build_gas_input_locations: remove unnecessary [0] with new storage plugin --- scripts/build_gas_input_locations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_gas_input_locations.py b/scripts/build_gas_input_locations.py index 6543d950..67dbc986 100644 --- a/scripts/build_gas_input_locations.py +++ b/scripts/build_gas_input_locations.py @@ -25,7 +25,7 @@ def read_scigrid_gas(fn): def build_gem_lng_data(fn): - df = pd.read_excel(fn[0], sheet_name="LNG terminals - data") + df = pd.read_excel(fn, sheet_name="LNG terminals - data") df = df.set_index("ComboID") remove_country = ["Cyprus", "Turkey"] # noqa: F841 @@ -46,7 +46,7 @@ def build_gem_lng_data(fn): def build_gem_prod_data(fn): - df = pd.read_excel(fn[0], sheet_name="Gas extraction - main") + df = pd.read_excel(fn, sheet_name="Gas extraction - main") df = df.set_index("GEM Unit ID") remove_country = ["Cyprus", "Türkiye"] # noqa: F841 @@ -60,7 +60,7 @@ def build_gem_prod_data(fn): & ~Longitude.isna()" ).copy() - p = pd.read_excel(fn[0], sheet_name="Gas extraction - production") + p = pd.read_excel(fn, sheet_name="Gas extraction - production") p = p.set_index("GEM Unit ID") p = p[p["Fuel description"] == "gas"] From e96a9896cbbc62d3d84d241379964d44ad4cb6e0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 11 Mar 2024 08:59:11 +0100 Subject: [PATCH 20/21] add release notes --- doc/release_notes.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 4500cbc9..9b494b32 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,11 @@ Release Notes Upcoming Release ================ +* Upgrade to Snakemake v8.5+. This version is the new minimum version required. + To upgrade an existing environment, run ``conda install -c bioconda + snakemake-minimal">=8.5"`` and ``pip install snakemake-storage-plugin-http`` + (https://github.com/PyPSA/pypsa-eur/pull/825). + * Corrected a bug leading to power plants operating after their DateOut (https://github.com/PyPSA/pypsa-eur/pull/958). Added additional grouping years before 1980. From e19e38b149f72ea2bed44f0ce5cf7f30cce95cad Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 11 Mar 2024 08:59:26 +0100 Subject: [PATCH 21/21] update version requirements for retrieve.yaml --- envs/retrieve.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/envs/retrieve.yaml b/envs/retrieve.yaml index b5db795d..a3c4828c 100644 --- a/envs/retrieve.yaml +++ b/envs/retrieve.yaml @@ -8,6 +8,11 @@ channels: - bioconda dependencies: - python>=3.8 -- snakemake-minimal>=7.7.0,<8.0.0 +- pip +- snakemake-minimal>=8.5 - pandas>=2.1 - tqdm +- pip: + - snakemake-storage-plugin-http + - snakemake-executor-plugin-slurm + - snakemake-executor-plugin-cluster-generic