From c0d1656091d50eaf347967bdf26803a0b8fc6904 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Wed, 10 Nov 2021 10:37:09 +0100
Subject: [PATCH] build_load_data: separate retrieving and processing load data

---
 Snakefile                  | 7 +++++++
 config.default.yaml        | 1 -
 config.tutorial.yaml       | 1 -
 scripts/build_load_data.py | 4 +---
 test/config.test1.yaml     | 1 -
 5 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/Snakefile b/Snakefile
index 1196d47b..cb50e3bf 100644
--- a/Snakefile
+++ b/Snakefile
@@ -67,7 +67,14 @@ if config['enable'].get('retrieve_databundle', True):
         script: 'scripts/retrieve_databundle.py'
 
 
+rule retrieve_load_data:
+    input: HTTP.remote("data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv", keep_local=True, static=True)
+    output: "data/load_raw.csv"
+    shell: "mv {input} {output}"
+
+
 rule build_load_data:
+    input: "data/load_raw.csv"
     output: "resources/load.csv"
     log: "logs/build_load_data.log"
     script: 'scripts/build_load_data.py'
diff --git a/config.default.yaml b/config.default.yaml
index a0ffbe47..f70e7c2c 100755
--- a/config.default.yaml
+++ b/config.default.yaml
@@ -188,7 +188,6 @@ transformers:
   type: ''
 
 load:
-  url: https://data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv
   power_statistics: True # only for files from <2019; set false in order to get ENTSOE transparency data 
   interpolate_limit: 3 # data gaps up until this size are interpolated linearly
   time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from 
diff --git a/config.tutorial.yaml b/config.tutorial.yaml
index c199712a..26ead242 100755
--- a/config.tutorial.yaml
+++ b/config.tutorial.yaml
@@ -150,7 +150,6 @@ transformers:
   type: ''
 
 load:
-  url: https://data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv
   power_statistics: True # only for files from <2019; set false in order to get ENTSOE transparency data 
   interpolate_limit: 3 # data gaps up until this size are interpolated linearly
   time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from 
diff --git a/scripts/build_load_data.py b/scripts/build_load_data.py
index b286df45..f71be6ea 100755
--- a/scripts/build_load_data.py
+++ b/scripts/build_load_data.py
@@ -14,7 +14,6 @@ Relevant Settings
     snapshots:
 
     load:
-        url:
         interpolate_limit:
         time_shift_for_large_gaps:
         manual_adjustments:
@@ -199,14 +198,13 @@ if __name__ == "__main__":
 
     config = snakemake.config
     powerstatistics = config['load']['power_statistics']
-    url = config['load']['url']
     interpolate_limit = config['load']['interpolate_limit']
     countries = config['countries']
     snapshots = pd.date_range(freq='h', **config['snapshots'])
     years = slice(snapshots[0], snapshots[-1])
     time_shift = config['load']['time_shift_for_large_gaps']
 
-    load = load_timeseries(url, years, countries, powerstatistics)
+    load = load_timeseries(snakemake.input[0], years, countries, powerstatistics)
 
     if config['load']['manual_adjustments']:
         load = manual_adjustment(load, powerstatistics)
diff --git a/test/config.test1.yaml b/test/config.test1.yaml
index 6b3e667a..2986037b 100755
--- a/test/config.test1.yaml
+++ b/test/config.test1.yaml
@@ -149,7 +149,6 @@ transformers:
   type: ''
 
 load:
-  url: https://data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv
   power_statistics: True # only for files from <2019; set false in order to get ENTSOE transparency data
   interpolate_limit: 3 # data gaps up until this size are interpolated linearly
   time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from