From 30a81a559f499aa4f1e29682269c838f3df3bd9b Mon Sep 17 00:00:00 2001 From: euronion <42553970+euronion@users.noreply.github.com> Date: Thu, 13 Jul 2023 08:22:22 +0200 Subject: [PATCH 1/6] Add test and option to disable online retrieve rules --- Snakefile | 5 ++++- config/config.default.yaml | 1 + rules/common.smk | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index 27ed4dfb..048bae28 100644 --- a/Snakefile +++ b/Snakefile @@ -48,7 +48,10 @@ wildcard_constraints: include: "rules/common.smk" include: "rules/collect.smk" -include: "rules/retrieve.smk" +if (config["enable"].get("retrieve","auto") == "auto" and has_internet_access())or config["enable"]["retrieve"] is True: + include: "rules/retrieve.smk" +else: + print("Datafile downloads disabled in config[retrieve] or no internet access.") include: "rules/build_electricity.smk" include: "rules/build_sector.smk" include: "rules/solve_electricity.smk" diff --git a/config/config.default.yaml b/config/config.default.yaml index c4b2620f..2e2c15dd 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -67,6 +67,7 @@ snapshots: inclusive: 'left' # include start, not end enable: + retrieve: auto prepare_links_p_nom: false retrieve_databundle: true retrieve_sector_databundle: true diff --git a/rules/common.smk b/rules/common.smk index 8f0c7cbb..3fc2c721 100644 --- a/rules/common.smk +++ b/rules/common.smk @@ -23,6 +23,22 @@ def memory(w): return int(factor * (10000 + 195 * int(w.clusters))) +# Check if the workflow has access to the internet by trying to access the HEAD of specified url +def has_internet_access(url="www.zenodo.org") -> bool: + import http.client as http_client + + # based on answer and comments from + # https://stackoverflow.com/a/29854274/11318472 + conn = http_client.HTTPConnection(url, timeout=5) # need access to zenodo anyway + try: + conn.request("HEAD", "/") + return True + except: + return False + finally: + conn.close() + + def input_eurostat(w): # 2016 includes BA, 2017 does not report_year = config["energy"]["eurostat_report_year"] From 730b32059696a9b19709b98243bba0c537d26587 Mon Sep 17 00:00:00 2001 From: euronion <42553970+euronion@users.noreply.github.com> Date: Fri, 14 Jul 2023 08:53:39 +0200 Subject: [PATCH 2/6] Move logic to retrieve.smk --- Snakefile | 5 +-- rules/retrieve.smk | 93 +++++++++++++++++++++++++++------------------- 2 files changed, 56 insertions(+), 42 deletions(-) diff --git a/Snakefile b/Snakefile index 048bae28..27ed4dfb 100644 --- a/Snakefile +++ b/Snakefile @@ -48,10 +48,7 @@ wildcard_constraints: include: "rules/common.smk" include: "rules/collect.smk" -if (config["enable"].get("retrieve","auto") == "auto" and has_internet_access())or config["enable"]["retrieve"] is True: - include: "rules/retrieve.smk" -else: - print("Datafile downloads disabled in config[retrieve] or no internet access.") +include: "rules/retrieve.smk" include: "rules/build_electricity.smk" include: "rules/build_sector.smk" include: "rules/solve_electricity.smk" diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 0a96406a..a253ec0c 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -2,7 +2,14 @@ # # SPDX-License-Identifier: MIT -if config["enable"].get("retrieve_databundle", True): +if config["enable"].get("retrieve", "auto") == "auto": + config["enable"]["retrieve"] = has_internet_access() + +if config["enable"]["retrieve"] is False: + print("Datafile downloads disabled in config[retrieve] or no internet access.") + + +if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", True): datafiles = [ "ch_cantons.csv", "je-e-21.03.02.xls", @@ -32,7 +39,7 @@ if config["enable"].get("retrieve_databundle", True): "../scripts/retrieve_databundle.py" -if config["enable"].get("retrieve_cutout", True): +if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True): rule retrieve_cutout: input: @@ -51,7 +58,7 @@ if config["enable"].get("retrieve_cutout", True): move(input[0], output[0]) -if config["enable"].get("retrieve_cost_data", True): +if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", True): rule retrieve_cost_data: input: @@ -73,7 +80,9 @@ if config["enable"].get("retrieve_cost_data", True): move(input[0], output[0]) -if config["enable"].get("retrieve_natura_raster", True): +if config["enable"]["retrieve"] and config["enable"].get( + "retrieve_natura_raster", True +): rule retrieve_natura_raster: input: @@ -93,7 +102,9 @@ if config["enable"].get("retrieve_natura_raster", True): move(input[0], output[0]) -if config["enable"].get("retrieve_sector_databundle", True): +if config["enable"]["retrieve"] and config["enable"].get( + "retrieve_sector_databundle", True +): datafiles = [ "data/eea/UNFCCC_v23.csv", "data/switzerland-sfoe/switzerland-new_format.csv", @@ -120,7 +131,9 @@ if config["enable"].get("retrieve_sector_databundle", True): "../scripts/retrieve_sector_databundle.py" -if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: +if config["enable"]["retrieve"] and ( + config["sector"]["gas_network"] or config["sector"]["H2_retrofit"] +): datafiles = [ "IGGIELGN_LNGs.geojson", "IGGIELGN_BorderPoints.geojson", @@ -140,37 +153,41 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: "../scripts/retrieve_gas_infrastructure_data.py" -rule retrieve_electricity_demand: - input: - HTTP.remote( - "data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv", - keep_local=True, - static=True, - ), - output: - "data/load_raw.csv", - log: - LOGS + "retrieve_electricity_demand.log", - resources: - mem_mb=5000, - retries: 2 - run: - move(input[0], output[0]) +if config["enable"]["retrieve"]: + + rule retrieve_electricity_demand: + input: + HTTP.remote( + "data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv", + keep_local=True, + static=True, + ), + output: + "data/load_raw.csv", + log: + LOGS + "retrieve_electricity_demand.log", + resources: + mem_mb=5000, + retries: 2 + run: + move(input[0], output[0]) -rule retrieve_ship_raster: - input: - HTTP.remote( - "https://zenodo.org/record/6953563/files/shipdensity_global.zip", - keep_local=True, - static=True, - ), - output: - "data/shipdensity_global.zip", - log: - LOGS + "retrieve_ship_raster.log", - resources: - mem_mb=5000, - retries: 2 - run: - move(input[0], output[0]) +if config["enable"]["retrieve"]: + + rule retrieve_ship_raster: + input: + HTTP.remote( + "https://zenodo.org/record/6953563/files/shipdensity_global.zip", + keep_local=True, + static=True, + ), + output: + "data/shipdensity_global.zip", + log: + LOGS + "retrieve_ship_raster.log", + resources: + mem_mb=5000, + retries: 2 + run: + move(input[0], output[0]) From 328e413b12d7dcf107e9395c1d64ff4362da6ff5 Mon Sep 17 00:00:00 2001 From: euronion <42553970+euronion@users.noreply.github.com> Date: Fri, 14 Jul 2023 13:41:22 +0200 Subject: [PATCH 3/6] Update release_notes.rst --- doc/release_notes.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index b7fcf2fa..2e114b1a 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -40,6 +40,10 @@ Upcoming Release e.g. by setting ``solving: options: transmission_losses: 2`` for an approximation with two tangents. +* Added configuration option ``enable: retrieve:`` to control whether data + retrieval rules from snakemake are enabled or not. Th default setting ``auto`` + will automatically detect and enable/disable the rules based on internet connectivity. + PyPSA-Eur 0.8.0 (18th March 2023) ================================= From 7437f32c0fd3c9d0aec994c2891b02f8f4ec6ec2 Mon Sep 17 00:00:00 2001 From: euronion <42553970+euronion@users.noreply.github.com> Date: Fri, 14 Jul 2023 13:41:25 +0200 Subject: [PATCH 4/6] Update enable.csv --- doc/configtables/enable.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/configtables/enable.csv b/doc/configtables/enable.csv index 8a543b46..e1349fef 100644 --- a/doc/configtables/enable.csv +++ b/doc/configtables/enable.csv @@ -1,4 +1,5 @@ ,Unit,Values,Description +enable,str or bool,"{auto, true, false}","Switch to include (true) or exclude (false) the retrieve_* rules of snakemake into the workflow; 'auto' sets true|false based on availability of an internet connection to prevent issues with snakemake failing due to lack of internet connection." prepare_links_p_nom,bool,"{true, false}","Switch to retrieve current HVDC projects from `Wikipedia `_" retrieve_databundle,bool,"{true, false}","Switch to retrieve databundle from zenodo via the rule :mod:`retrieve_databundle` or whether to keep a custom databundle located in the corresponding folder." retrieve_sector_databundle,bool,"{true, false}","Switch to retrieve sector databundle from zenodo via the rule :mod:`retrieve_sector_databundle` or whether to keep a custom databundle located in the corresponding folder." From aa7a2c823acec4d805f08e92e34e48f740cd32a5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 14 Jul 2023 11:42:06 +0000 Subject: [PATCH 5/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- doc/release_notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 2e114b1a..4c7219e7 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -41,7 +41,7 @@ Upcoming Release approximation with two tangents. * Added configuration option ``enable: retrieve:`` to control whether data - retrieval rules from snakemake are enabled or not. Th default setting ``auto`` + retrieval rules from snakemake are enabled or not. Th default setting ``auto`` will automatically detect and enable/disable the rules based on internet connectivity. From e5ce76bf480670940cc6020ef76f886e325bd1ac Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 17 Jul 2023 23:29:16 +0000 Subject: [PATCH 6/6] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/docformatter: v1.7.4 → v1.7.5](https://github.com/PyCQA/docformatter/compare/v1.7.4...v1.7.5) - [github.com/psf/black: 23.3.0 → 23.7.0](https://github.com/psf/black/compare/23.3.0...23.7.0) - [github.com/macisamuele/language-formatters-pre-commit-hooks: v2.9.0 → v2.10.0](https://github.com/macisamuele/language-formatters-pre-commit-hooks/compare/v2.9.0...v2.10.0) --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cf811240..6fb0dbf0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: # Make docstrings PEP 257 compliant - repo: https://github.com/PyCQA/docformatter - rev: v1.7.4 + rev: v1.7.5 hooks: - id: docformatter args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"] @@ -51,7 +51,7 @@ repos: # Formatting with "black" coding style - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.7.0 hooks: # Format Python files - id: black @@ -67,7 +67,7 @@ repos: # Do YAML formatting (before the linter checks it for misses) - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks - rev: v2.9.0 + rev: v2.10.0 hooks: - id: pretty-format-yaml args: [--autofix, --indent, "2", --preserve-quotes]