Merge branch 'master' into validation

.pre-commit-config.yaml

@@ -39,7 +39,7 @@ repos:
 
   # Make docstrings PEP 257 compliant
 - repo: https://github.com/PyCQA/docformatter
-  rev: v1.7.4
+  rev: v1.7.5
   hooks:
   - id: docformatter
     args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"]
@@ -51,7 +51,7 @@ repos:
 
   # Formatting with "black" coding style
 - repo: https://github.com/psf/black
-  rev: 23.3.0
+  rev: 23.7.0
   hooks:
       # Format Python files
   - id: black
@@ -67,7 +67,7 @@ repos:
 
   # Do YAML formatting (before the linter checks it for misses)
 - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
-  rev: v2.9.0
+  rev: v2.10.0
   hooks:
   - id: pretty-format-yaml
     args: [--autofix, --indent, "2", --preserve-quotes]

config/config.default.yaml

@@ -62,6 +62,7 @@ snapshots:
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#enable
 enable:
+  retrieve: auto
   prepare_links_p_nom: false
   retrieve_databundle: true
   retrieve_sector_databundle: true

doc/configtables/enable.csv

@@ -1,4 +1,5 @@
 ,Unit,Values,Description
+enable,str or bool,"{auto, true, false}","Switch to include (true) or exclude (false) the retrieve_* rules of snakemake into the workflow; 'auto' sets true|false based on availability of an internet connection to prevent issues with snakemake failing due to lack of internet connection."
 prepare_links_p_nom,bool,"{true, false}","Switch to retrieve current HVDC projects from `Wikipedia <https://en.wikipedia.org/wiki/List_of_HVDC_projects>`_"
 retrieve_databundle,bool,"{true, false}","Switch to retrieve databundle from zenodo via the rule :mod:`retrieve_databundle` or whether to keep a custom databundle located in the corresponding folder."
 retrieve_sector_databundle,bool,"{true, false}","Switch to retrieve sector databundle from zenodo via the rule :mod:`retrieve_sector_databundle` or whether to keep a custom databundle located in the corresponding folder."

doc/release_notes.rst

@@ -40,6 +40,10 @@ Upcoming Release
   e.g. by setting ``solving: options: transmission_losses: 2`` for an
   approximation with two tangents.
 
+* Added configuration option ``enable: retrieve:`` to control whether data
+  retrieval rules from snakemake are enabled or not. Th default setting ``auto``
+  will automatically detect and enable/disable the rules based on internet connectivity.
+
 
 PyPSA-Eur 0.8.0 (18th March 2023)
 =================================

rules/common.smk

@@ -23,6 +23,22 @@ def memory(w):
         return int(factor * (10000 + 195 * int(w.clusters)))
 
 
+# Check if the workflow has access to the internet by trying to access the HEAD of specified url
+def has_internet_access(url="www.zenodo.org") -> bool:
+    import http.client as http_client
+
+    # based on answer and comments from
+    # https://stackoverflow.com/a/29854274/11318472
+    conn = http_client.HTTPConnection(url, timeout=5)  # need access to zenodo anyway
+    try:
+        conn.request("HEAD", "/")
+        return True
+    except:
+        return False
+    finally:
+        conn.close()
+
+
 def input_eurostat(w):
     # 2016 includes BA, 2017 does not
     report_year = config["energy"]["eurostat_report_year"]

rules/retrieve.smk

@@ -2,7 +2,14 @@
 #
 # SPDX-License-Identifier: MIT
 
-if config["enable"].get("retrieve_databundle", True):
+if config["enable"].get("retrieve", "auto") == "auto":
+    config["enable"]["retrieve"] = has_internet_access()
+
+if config["enable"]["retrieve"] is False:
+    print("Datafile downloads disabled in config[retrieve] or no internet access.")
+
+
+if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", True):
     datafiles = [
         "ch_cantons.csv",
         "je-e-21.03.02.xls",
@@ -32,7 +39,7 @@ if config["enable"].get("retrieve_databundle", True):
             "../scripts/retrieve_databundle.py"
 
 
-if config["enable"].get("retrieve_cutout", True):
+if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True):
 
     rule retrieve_cutout:
         input:
@@ -51,7 +58,7 @@ if config["enable"].get("retrieve_cutout", True):
             move(input[0], output[0])
 
 
-if config["enable"].get("retrieve_cost_data", True):
+if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", True):
 
     rule retrieve_cost_data:
         input:
@@ -73,7 +80,9 @@ if config["enable"].get("retrieve_cost_data", True):
             move(input[0], output[0])
 
 
-if config["enable"].get("retrieve_natura_raster", True):
+if config["enable"]["retrieve"] and config["enable"].get(
+    "retrieve_natura_raster", True
+):
 
     rule retrieve_natura_raster:
         input:
@@ -93,7 +102,9 @@ if config["enable"].get("retrieve_natura_raster", True):
             move(input[0], output[0])
 
 
-if config["enable"].get("retrieve_sector_databundle", True):
+if config["enable"]["retrieve"] and config["enable"].get(
+    "retrieve_sector_databundle", True
+):
     datafiles = [
         "data/eea/UNFCCC_v23.csv",
         "data/switzerland-sfoe/switzerland-new_format.csv",
@@ -120,7 +131,9 @@ if config["enable"].get("retrieve_sector_databundle", True):
             "../scripts/retrieve_sector_databundle.py"
 
 
-if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
+if config["enable"]["retrieve"] and (
+    config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]
+):
     datafiles = [
         "IGGIELGN_LNGs.geojson",
         "IGGIELGN_BorderPoints.geojson",
@@ -140,10 +153,12 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
             "../scripts/retrieve_gas_infrastructure_data.py"
 
 
-rule retrieve_electricity_demand:
+if config["enable"]["retrieve"]:
+
+    rule retrieve_electricity_demand:
         input:
             HTTP.remote(
-            "data.open-power-system-data.org/time_series/{version}/time_series_60min_singleindex.csv".format(
+                "data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv".format(
             version="2019-06-05"
                 if config["snapshots"]["end"] < "2019"
                 else "latest"
@@ -162,7 +177,9 @@ rule retrieve_electricity_demand:
             move(input[0], output[0])
 
 
-rule retrieve_ship_raster:
+if config["enable"]["retrieve"]:
+
+    rule retrieve_ship_raster:
         input:
             HTTP.remote(
                 "https://zenodo.org/record/6953563/files/shipdensity_global.zip",