From 48850431fbc296d4178933ea0f869bc515068b92 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 30 Dec 2023 14:37:45 +0100 Subject: [PATCH] Snakefile: switch to new storage providers --- Snakefile | 4 ---- envs/environment.yaml | 1 + rules/build_sector.smk | 14 +++++-------- rules/retrieve.smk | 47 ++++++++++++++---------------------------- 4 files changed, 21 insertions(+), 45 deletions(-) diff --git a/Snakefile b/Snakefile index 83530df7..42988ebc 100644 --- a/Snakefile +++ b/Snakefile @@ -5,10 +5,6 @@ from os.path import normpath, exists from shutil import copyfile, move, rmtree -from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider - -HTTP = HTTPRemoteProvider() - from snakemake.utils import min_version min_version("7.7") diff --git a/envs/environment.yaml b/envs/environment.yaml index b9f9ac99..ce733ed9 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -59,3 +59,4 @@ dependencies: - pip: - tsam>=2.3.1 + - snakemake-storage-plugin-http \ No newline at end of file diff --git a/rules/build_sector.smk b/rules/build_sector.smk index dd49fc6f..3f71ba72 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -85,9 +85,8 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: rule build_gas_input_locations: input: - lng=HTTP.remote( + lng=storage( "https://globalenergymonitor.org/wp-content/uploads/2023/07/Europe-Gas-Tracker-2023-03-v3.xlsx", - keep_local=True, ), entry="data/gas_network/scigrid-gas/data/IGGIELGN_BorderPoints.geojson", production="data/gas_network/scigrid-gas/data/IGGIELGN_Productions.geojson", @@ -268,9 +267,8 @@ rule build_biomass_potentials: params: biomass=config["biomass"], input: - enspreso_biomass=HTTP.remote( + enspreso_biomass=storage( "https://zenodo.org/records/10356004/files/ENSPRESO_BIOMASS.xlsx", - keep_local=True, ), nuts2="data/bundle-sector/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21 regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -300,9 +298,8 @@ if config["sector"]["biomass_transport"] or config["sector"]["biomass_spatial"]: rule build_biomass_transport_costs: input: - transport_cost_data=HTTP.remote( - "publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass potentials in europe_web rev.pdf", - keep_local=True, + transport_cost_data=storage( + "https://publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass potentials in europe_web rev.pdf", ), output: biomass_transport_costs=RESOURCES + "biomass_transport_costs.csv", @@ -334,9 +331,8 @@ if config["sector"]["regional_co2_sequestration_potential"]["enable"]: "regional_co2_sequestration_potential" ], input: - sequestration_potential=HTTP.remote( + sequestration_potential=storage( "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson", - keep_local=True, ), regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 0483a833..a35cc53c 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -64,9 +64,8 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True rule retrieve_cutout: input: - HTTP.remote( - "zenodo.org/record/6382570/files/{cutout}.nc", - static=True, + storage( + "https://zenodo.org/record/6382570/files/{cutout}.nc", ), output: protected("cutouts/" + CDIR + "{cutout}.nc"), @@ -83,12 +82,11 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", T rule retrieve_cost_data: input: - HTTP.remote( - "raw.githubusercontent.com/PyPSA/technology-data/{}/outputs/".format( + storage( + "https://raw.githubusercontent.com/PyPSA/technology-data/{}/outputs/".format( config["costs"]["version"] ) + "costs_{year}.csv", - keep_local=True, ), output: "data/costs_{year}.csv", @@ -107,10 +105,8 @@ if config["enable"]["retrieve"] and config["enable"].get( rule retrieve_natura_raster: input: - HTTP.remote( - "zenodo.org/record/4706686/files/natura.tiff", - keep_local=True, - static=True, + storage( + "https://zenodo.org/record/4706686/files/natura.tiff", ), output: RESOURCES + "natura.tiff", @@ -188,14 +184,12 @@ if config["enable"]["retrieve"]: rule retrieve_electricity_demand: input: - HTTP.remote( - "data.open-power-system-data.org/time_series/{version}/time_series_60min_singleindex.csv".format( + storage( + "https://data.open-power-system-data.org/time_series/{version}/time_series_60min_singleindex.csv".format( version="2019-06-05" if config["snapshots"]["end"] < "2019" else "2020-10-06" ), - keep_local=True, - static=True, ), output: RESOURCES + "load_raw.csv", @@ -212,10 +206,8 @@ if config["enable"]["retrieve"]: rule retrieve_ship_raster: input: - HTTP.remote( + storage( "https://zenodo.org/record/6953563/files/shipdensity_global.zip", - keep_local=True, - static=True, ), output: protected("data/shipdensity_global.zip"), @@ -234,9 +226,8 @@ if config["enable"]["retrieve"]: # Website: https://land.copernicus.eu/global/products/lc rule download_copernicus_land_cover: input: - HTTP.remote( - "zenodo.org/record/3939050/files/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif", - static=True, + storage( + "https://zenodo.org/record/3939050/files/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif", ), output: "data/Copernicus_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif", @@ -279,11 +270,7 @@ if config["enable"]["retrieve"]: # Website: https://www.protectedplanet.net/en/thematic-areas/wdpa rule download_wdpa: input: - HTTP.remote( - url, - static=True, - keep_local=True, - ), + storage(url), params: zip="data/WDPA_shp.zip", folder=directory("data/WDPA"), @@ -305,10 +292,8 @@ if config["enable"]["retrieve"]: # extract the main zip and then merge the contained 3 zipped shapefiles # Website: https://www.protectedplanet.net/en/thematic-areas/marine-protected-areas input: - HTTP.remote( - f"d1gam3xoknrgr2.cloudfront.net/current/WDPA_WDOECM_{bYYYY}_Public_marine_shp.zip", - static=True, - keep_local=True, + storage( + f"https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_WDOECM_{bYYYY}_Public_marine_shp.zip", ), params: zip="data/WDPA_WDOECM_marine.zip", @@ -330,10 +315,8 @@ if config["enable"]["retrieve"]: rule retrieve_monthly_co2_prices: input: - HTTP.remote( + storage( "https://www.eex.com/fileadmin/EEX/Downloads/EUA_Emission_Spot_Primary_Market_Auction_Report/Archive_Reports/emission-spot-primary-market-auction-report-2019-data.xls", - keep_local=True, - static=True, ), output: "data/validation/emission-spot-primary-market-auction-report-2019-data.xls",