bjoern/pypsa-eur
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Snakefile: switch to new storage providers

Browse Source
This commit is contained in:
Fabian Neumann 2023-12-30 14:37:45 +01:00
parent 4349903a72
commit 48850431fb
4 changed files with 21 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Snakefile
View File

@ -5,10 +5,6 @@
from os.path import normpath, exists
from shutil import copyfile, move, rmtree
from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider
HTTP = HTTPRemoteProvider()
from snakemake.utils import min_version
min_version("7.7")

1
envs/environment.yaml
View File

@ -59,3 +59,4 @@ dependencies:
- pip:
- tsam>=2.3.1
- snakemake-storage-plugin-http

14
rules/build_sector.smk
View File

@ -85,9 +85,8 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
rule build_gas_input_locations:
input:
lng=HTTP.remote(
lng=storage(
"https://globalenergymonitor.org/wp-content/uploads/2023/07/Europe-Gas-Tracker-2023-03-v3.xlsx",
keep_local=True,
),
entry="data/gas_network/scigrid-gas/data/IGGIELGN_BorderPoints.geojson",
production="data/gas_network/scigrid-gas/data/IGGIELGN_Productions.geojson",
@ -268,9 +267,8 @@ rule build_biomass_potentials:
params:
biomass=config["biomass"],
input:
enspreso_biomass=HTTP.remote(
enspreso_biomass=storage(
"https://zenodo.org/records/10356004/files/ENSPRESO_BIOMASS.xlsx",
keep_local=True,
),
nuts2="data/bundle-sector/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21
regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
@ -300,9 +298,8 @@ if config["sector"]["biomass_transport"] or config["sector"]["biomass_spatial"]:
rule build_biomass_transport_costs:
input:
transport_cost_data=HTTP.remote(
"publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass potentials in europe_web rev.pdf",
keep_local=True,
transport_cost_data=storage(
"https://publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass potentials in europe_web rev.pdf",
),
output:
biomass_transport_costs=RESOURCES + "biomass_transport_costs.csv",
@ -334,9 +331,8 @@ if config["sector"]["regional_co2_sequestration_potential"]["enable"]:
"regional_co2_sequestration_potential"
],
input:
sequestration_potential=HTTP.remote(
sequestration_potential=storage(
"https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson",
keep_local=True,
),
regions_onshore=RESOURCES
+ "regions_onshore_elec_s{simpl}_{clusters}.geojson",

47
rules/retrieve.smk
View File

@ -64,9 +64,8 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True
rule retrieve_cutout:
input:
HTTP.remote(
"zenodo.org/record/6382570/files/{cutout}.nc",
static=True,
storage(
"https://zenodo.org/record/6382570/files/{cutout}.nc",
),
output:
protected("cutouts/" + CDIR + "{cutout}.nc"),
@ -83,12 +82,11 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", T
rule retrieve_cost_data:
input:
HTTP.remote(
"raw.githubusercontent.com/PyPSA/technology-data/{}/outputs/".format(
storage(
"https://raw.githubusercontent.com/PyPSA/technology-data/{}/outputs/".format(
config["costs"]["version"]
)
+ "costs_{year}.csv",
keep_local=True,
),
output:
"data/costs_{year}.csv",
@ -107,10 +105,8 @@ if config["enable"]["retrieve"] and config["enable"].get(
rule retrieve_natura_raster:
input:
HTTP.remote(
"zenodo.org/record/4706686/files/natura.tiff",
keep_local=True,
static=True,
storage(
"https://zenodo.org/record/4706686/files/natura.tiff",
),
output:
RESOURCES + "natura.tiff",
@ -188,14 +184,12 @@ if config["enable"]["retrieve"]:
rule retrieve_electricity_demand:
input:
HTTP.remote(
"data.open-power-system-data.org/time_series/{version}/time_series_60min_singleindex.csv".format(
storage(
"https://data.open-power-system-data.org/time_series/{version}/time_series_60min_singleindex.csv".format(
version="2019-06-05"
if config["snapshots"]["end"] < "2019"
else "2020-10-06"
),
keep_local=True,
static=True,
),
output:
RESOURCES + "load_raw.csv",
@ -212,10 +206,8 @@ if config["enable"]["retrieve"]:
rule retrieve_ship_raster:
input:
HTTP.remote(
storage(
"https://zenodo.org/record/6953563/files/shipdensity_global.zip",
keep_local=True,
static=True,
),
output:
protected("data/shipdensity_global.zip"),
@ -234,9 +226,8 @@ if config["enable"]["retrieve"]:
# Website: https://land.copernicus.eu/global/products/lc
rule download_copernicus_land_cover:
input:
HTTP.remote(
"zenodo.org/record/3939050/files/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif",
static=True,
storage(
"https://zenodo.org/record/3939050/files/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif",
),
output:
"data/Copernicus_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif",
@ -279,11 +270,7 @@ if config["enable"]["retrieve"]:
# Website: https://www.protectedplanet.net/en/thematic-areas/wdpa
rule download_wdpa:
input:
HTTP.remote(
url,
static=True,
keep_local=True,
),
storage(url),
params:
zip="data/WDPA_shp.zip",
folder=directory("data/WDPA"),
@ -305,10 +292,8 @@ if config["enable"]["retrieve"]:
# extract the main zip and then merge the contained 3 zipped shapefiles
# Website: https://www.protectedplanet.net/en/thematic-areas/marine-protected-areas
input:
HTTP.remote(
f"d1gam3xoknrgr2.cloudfront.net/current/WDPA_WDOECM_{bYYYY}_Public_marine_shp.zip",
static=True,
keep_local=True,
storage(
f"https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_WDOECM_{bYYYY}_Public_marine_shp.zip",
),
params:
zip="data/WDPA_WDOECM_marine.zip",
@ -330,10 +315,8 @@ if config["enable"]["retrieve"]:
rule retrieve_monthly_co2_prices:
input:
HTTP.remote(
storage(
"https://www.eex.com/fileadmin/EEX/Downloads/EUA_Emission_Spot_Primary_Market_Auction_Report/Archive_Reports/emission-spot-primary-market-auction-report-2019-data.xls",
keep_local=True,
static=True,
),
output:
"data/validation/emission-spot-primary-market-auction-report-2019-data.xls",