diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cf811240..6fb0dbf0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: # Make docstrings PEP 257 compliant - repo: https://github.com/PyCQA/docformatter - rev: v1.7.4 + rev: v1.7.5 hooks: - id: docformatter args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"] @@ -51,7 +51,7 @@ repos: # Formatting with "black" coding style - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.7.0 hooks: # Format Python files - id: black @@ -67,7 +67,7 @@ repos: # Do YAML formatting (before the linter checks it for misses) - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks - rev: v2.9.0 + rev: v2.10.0 hooks: - id: pretty-format-yaml args: [--autofix, --indent, "2", --preserve-quotes] diff --git a/config/config.default.yaml b/config/config.default.yaml index be03244c..2dd650b2 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -62,6 +62,7 @@ snapshots: # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#enable enable: + retrieve: auto prepare_links_p_nom: false retrieve_databundle: true retrieve_sector_databundle: true diff --git a/doc/configtables/enable.csv b/doc/configtables/enable.csv index 8a543b46..e1349fef 100644 --- a/doc/configtables/enable.csv +++ b/doc/configtables/enable.csv @@ -1,4 +1,5 @@ ,Unit,Values,Description +enable,str or bool,"{auto, true, false}","Switch to include (true) or exclude (false) the retrieve_* rules of snakemake into the workflow; 'auto' sets true|false based on availability of an internet connection to prevent issues with snakemake failing due to lack of internet connection." prepare_links_p_nom,bool,"{true, false}","Switch to retrieve current HVDC projects from `Wikipedia `_" retrieve_databundle,bool,"{true, false}","Switch to retrieve databundle from zenodo via the rule :mod:`retrieve_databundle` or whether to keep a custom databundle located in the corresponding folder." retrieve_sector_databundle,bool,"{true, false}","Switch to retrieve sector databundle from zenodo via the rule :mod:`retrieve_sector_databundle` or whether to keep a custom databundle located in the corresponding folder." diff --git a/doc/release_notes.rst b/doc/release_notes.rst index b7fcf2fa..4c7219e7 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -40,6 +40,10 @@ Upcoming Release e.g. by setting ``solving: options: transmission_losses: 2`` for an approximation with two tangents. +* Added configuration option ``enable: retrieve:`` to control whether data + retrieval rules from snakemake are enabled or not. Th default setting ``auto`` + will automatically detect and enable/disable the rules based on internet connectivity. + PyPSA-Eur 0.8.0 (18th March 2023) ================================= diff --git a/rules/common.smk b/rules/common.smk index c13e469e..ec5be355 100644 --- a/rules/common.smk +++ b/rules/common.smk @@ -23,6 +23,22 @@ def memory(w): return int(factor * (10000 + 195 * int(w.clusters))) +# Check if the workflow has access to the internet by trying to access the HEAD of specified url +def has_internet_access(url="www.zenodo.org") -> bool: + import http.client as http_client + + # based on answer and comments from + # https://stackoverflow.com/a/29854274/11318472 + conn = http_client.HTTPConnection(url, timeout=5) # need access to zenodo anyway + try: + conn.request("HEAD", "/") + return True + except: + return False + finally: + conn.close() + + def input_eurostat(w): # 2016 includes BA, 2017 does not report_year = config["energy"]["eurostat_report_year"] diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 2e076a75..a69158a5 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -2,7 +2,14 @@ # # SPDX-License-Identifier: MIT -if config["enable"].get("retrieve_databundle", True): +if config["enable"].get("retrieve", "auto") == "auto": + config["enable"]["retrieve"] = has_internet_access() + +if config["enable"]["retrieve"] is False: + print("Datafile downloads disabled in config[retrieve] or no internet access.") + + +if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", True): datafiles = [ "ch_cantons.csv", "je-e-21.03.02.xls", @@ -32,7 +39,7 @@ if config["enable"].get("retrieve_databundle", True): "../scripts/retrieve_databundle.py" -if config["enable"].get("retrieve_cutout", True): +if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True): rule retrieve_cutout: input: @@ -51,7 +58,7 @@ if config["enable"].get("retrieve_cutout", True): move(input[0], output[0]) -if config["enable"].get("retrieve_cost_data", True): +if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", True): rule retrieve_cost_data: input: @@ -73,7 +80,9 @@ if config["enable"].get("retrieve_cost_data", True): move(input[0], output[0]) -if config["enable"].get("retrieve_natura_raster", True): +if config["enable"]["retrieve"] and config["enable"].get( + "retrieve_natura_raster", True +): rule retrieve_natura_raster: input: @@ -93,7 +102,9 @@ if config["enable"].get("retrieve_natura_raster", True): move(input[0], output[0]) -if config["enable"].get("retrieve_sector_databundle", True): +if config["enable"]["retrieve"] and config["enable"].get( + "retrieve_sector_databundle", True +): datafiles = [ "data/eea/UNFCCC_v23.csv", "data/switzerland-sfoe/switzerland-new_format.csv", @@ -120,7 +131,9 @@ if config["enable"].get("retrieve_sector_databundle", True): "../scripts/retrieve_sector_databundle.py" -if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: +if config["enable"]["retrieve"] and ( + config["sector"]["gas_network"] or config["sector"]["H2_retrofit"] +): datafiles = [ "IGGIELGN_LNGs.geojson", "IGGIELGN_BorderPoints.geojson", @@ -140,41 +153,45 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: "../scripts/retrieve_gas_infrastructure_data.py" -rule retrieve_electricity_demand: - input: - HTTP.remote( - "data.open-power-system-data.org/time_series/{version}/time_series_60min_singleindex.csv".format( +if config["enable"]["retrieve"]: + + rule retrieve_electricity_demand: + input: + HTTP.remote( + "data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv".format( version="2019-06-05" if config["snapshots"]["end"] < "2019" else "latest" ), - keep_local=True, - static=True, - ), - output: - "data/load_raw.csv", - log: - LOGS + "retrieve_electricity_demand.log", - resources: - mem_mb=5000, - retries: 2 - run: - move(input[0], output[0]) + keep_local=True, + static=True, + ), + output: + "data/load_raw.csv", + log: + LOGS + "retrieve_electricity_demand.log", + resources: + mem_mb=5000, + retries: 2 + run: + move(input[0], output[0]) -rule retrieve_ship_raster: - input: - HTTP.remote( - "https://zenodo.org/record/6953563/files/shipdensity_global.zip", - keep_local=True, - static=True, - ), - output: - "data/shipdensity_global.zip", - log: - LOGS + "retrieve_ship_raster.log", - resources: - mem_mb=5000, - retries: 2 - run: - move(input[0], output[0]) +if config["enable"]["retrieve"]: + + rule retrieve_ship_raster: + input: + HTTP.remote( + "https://zenodo.org/record/6953563/files/shipdensity_global.zip", + keep_local=True, + static=True, + ), + output: + "data/shipdensity_global.zip", + log: + LOGS + "retrieve_ship_raster.log", + resources: + mem_mb=5000, + retries: 2 + run: + move(input[0], output[0])