56 lines
1.7 KiB
Python
56 lines
1.7 KiB
Python
# -*- coding: utf-8 -*-
|
|
# SPDX-FileCopyrightText: : 2024- The PyPSA-Eur Authors
|
|
#
|
|
# SPDX-License-Identifier: MIT
|
|
"""
|
|
Retrieve and extract JRC IDEES 2021 data.
|
|
"""
|
|
|
|
import os
|
|
import requests
|
|
from pathlib import Path
|
|
from bs4 import BeautifulSoup
|
|
from _helpers import configure_logging, progress_retrieve, set_scenario_config
|
|
import zipfile
|
|
import logging
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Define the base URL
|
|
url_jrc = "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/JRC-IDEES/JRC-IDEES-2021_v1/"
|
|
|
|
if __name__ == "__main__":
|
|
if "snakemake" not in globals():
|
|
from _helpers import mock_snakemake
|
|
snakemake = mock_snakemake("retrieve_jrc_idees")
|
|
rootpath = ".."
|
|
else:
|
|
rootpath = "."
|
|
|
|
configure_logging(snakemake)
|
|
set_scenario_config(snakemake)
|
|
|
|
disable_progress = snakemake.config["run"].get("disable_progressbar", False)
|
|
|
|
# create a local directory to save the zip files
|
|
local_dir = snakemake.output[0]
|
|
if not os.path.exists(local_dir):
|
|
os.makedirs(local_dir)
|
|
|
|
# get the list of zip files from the JRC URL
|
|
response = requests.get(url_jrc)
|
|
soup = BeautifulSoup(response.text, 'html.parser')
|
|
zip_files = [link.get('href') for link in soup.find_all('a')
|
|
if link.get('href').endswith('.zip')]
|
|
|
|
logger.info(f"Downloading {len(zip_files)} .zip files for JRC IDEES from '{url_jrc}'.")
|
|
|
|
# download and unpack each zip file
|
|
for zip_file in zip_files:
|
|
logger.info(f"Downloading and unpacking {zip_file}")
|
|
zip_url = url_jrc + zip_file
|
|
to_fn = local_dir+"/" + zip_file[:-4]
|
|
progress_retrieve(zip_url, to_fn, disable=disable_progress)
|
|
|