From 30a81a559f499aa4f1e29682269c838f3df3bd9b Mon Sep 17 00:00:00 2001
From: euronion <42553970+euronion@users.noreply.github.com>
Date: Thu, 13 Jul 2023 08:22:22 +0200
Subject: [PATCH] Add test and option to disable online retrieve rules

---
 Snakefile                  |  5 ++++-
 config/config.default.yaml |  1 +
 rules/common.smk           | 16 ++++++++++++++++
 3 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/Snakefile b/Snakefile
index 27ed4dfb..048bae28 100644
--- a/Snakefile
+++ b/Snakefile
@@ -48,7 +48,10 @@ wildcard_constraints:
 
 include: "rules/common.smk"
 include: "rules/collect.smk"
-include: "rules/retrieve.smk"
+if (config["enable"].get("retrieve","auto") == "auto" and has_internet_access())or config["enable"]["retrieve"] is True:
+    include: "rules/retrieve.smk"
+else:
+    print("Datafile downloads disabled in config[retrieve] or no internet access.")
 include: "rules/build_electricity.smk"
 include: "rules/build_sector.smk"
 include: "rules/solve_electricity.smk"
diff --git a/config/config.default.yaml b/config/config.default.yaml
index c4b2620f..2e2c15dd 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -67,6 +67,7 @@ snapshots:
   inclusive: 'left' # include start, not end
 
 enable:
+  retrieve: auto
   prepare_links_p_nom: false
   retrieve_databundle: true
   retrieve_sector_databundle: true
diff --git a/rules/common.smk b/rules/common.smk
index 8f0c7cbb..3fc2c721 100644
--- a/rules/common.smk
+++ b/rules/common.smk
@@ -23,6 +23,22 @@ def memory(w):
         return int(factor * (10000 + 195 * int(w.clusters)))
 
 
+# Check if the workflow has access to the internet by trying to access the HEAD of specified url
+def has_internet_access(url="www.zenodo.org") -> bool:
+    import http.client as http_client
+
+    # based on answer and comments from
+    # https://stackoverflow.com/a/29854274/11318472
+    conn = http_client.HTTPConnection(url, timeout=5)  # need access to zenodo anyway
+    try:
+        conn.request("HEAD", "/")
+        return True
+    except:
+        return False
+    finally:
+        conn.close()
+
+
 def input_eurostat(w):
     # 2016 includes BA, 2017 does not
     report_year = config["energy"]["eurostat_report_year"]