diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 8ef373d1..9250e745 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -142,9 +142,6 @@ if config["enable"]["retrieve"] and config["enable"].get( protected( directory("data/bundle-sector/eurostat-energy_balances-may_2018_edition") ), - protected( - directory("data/bundle-sector/eurostat-energy_balances-april_2023_edition") - ), protected(directory("data/bundle-sector/jrc-idees-2015")), ] @@ -160,6 +157,16 @@ if config["enable"]["retrieve"] and config["enable"].get( script: "../scripts/retrieve_sector_databundle.py" + rule retrieve_eurostat_data: + output: + protected( + directory("data/bundle-sector/eurostat-energy_balances-april_2023_edition") + ), + log: + "logs/retrieve_eurostat_data.log", + retries: 2 + script: + "../scripts/retrieve_eurostat_data.py" if config["enable"]["retrieve"]: datafiles = [ diff --git a/scripts/retrieve_eurostat_data.py b/scripts/retrieve_eurostat_data.py new file mode 100644 index 00000000..aedc43c9 --- /dev/null +++ b/scripts/retrieve_eurostat_data.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2024- The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT +""" +Retrieve and extract eurostat energy balances data. +""" + + +import logging +import zipfile +from pathlib import Path + +from _helpers import ( + configure_logging, + progress_retrieve, + set_scenario_config, +) + +logger = logging.getLogger(__name__) + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake("retrieve_eurostat_data") + rootpath = ".." + else: + rootpath = "." + configure_logging(snakemake) + set_scenario_config(snakemake) + + disable_progress = snakemake.config["run"].get("disable_progressbar", False) + url_eurostat = "https://ec.europa.eu/eurostat/documents/38154/4956218/Balances-December2022.zip/f7cf0d19-5c0f-60ad-4e48-098a5ddd6e48?t=1671184070589" + tarball_fn = Path(f"{rootpath}/data/bundle-sector/eurostat_2023.zip") + to_fn = Path(f"{rootpath}/data/bundle-sector/eurostat-energy_balances-april_2023_edition/") + + logger.info(f"Downloading Eurostat data from '{url_eurostat}'.") + progress_retrieve(url_eurostat, tarball_fn, disable=disable_progress) + + logger.info("Extracting Eurostat data.") + with zipfile.ZipFile(tarball_fn, 'r') as zip_ref: + zip_ref.extractall(to_fn) + + logger.info(f"Eurostat data available in '{to_fn}'.") \ No newline at end of file diff --git a/scripts/retrieve_sector_databundle.py b/scripts/retrieve_sector_databundle.py index a075119e..d311dac7 100644 --- a/scripts/retrieve_sector_databundle.py +++ b/scripts/retrieve_sector_databundle.py @@ -8,7 +8,6 @@ Retrieve and extract data bundle for sector-coupled studies. import logging import tarfile -import zipfile from pathlib import Path from _helpers import ( @@ -47,17 +46,4 @@ if __name__ == "__main__": tarball_fn.unlink() - logger.info(f"Databundle available in '{to_fn}'.") - - url_eurostat = "https://ec.europa.eu/eurostat/documents/38154/4956218/Balances-December2022.zip/f7cf0d19-5c0f-60ad-4e48-098a5ddd6e48?t=1671184070589" - tarball_fn = Path(f"{rootpath}/data/bundle-sector/eurostat_2023.zip") - to_fn = Path(f"{rootpath}/data/bundle-sector/eurostat-energy_balances-april_2023_edition/") - - logger.info(f"Downloading Eurostat data from '{url_eurostat}'.") - progress_retrieve(url_eurostat, tarball_fn, disable=disable_progress) - - logger.info("Extracting Eurostat data.") - with zipfile.ZipFile(tarball_fn, 'r') as zip_ref: - zip_ref.extractall(to_fn) - - logger.info(f"Eurostat data available in '{to_fn}'.") \ No newline at end of file + logger.info(f"Databundle available in '{to_fn}'.") \ No newline at end of file