build_load_data: separate retrieving and processing load data

Snakefile

@@ -67,7 +67,14 @@ if config['enable'].get('retrieve_databundle', True):
         script: 'scripts/retrieve_databundle.py'
 
 
+rule retrieve_load_data:
+    input: HTTP.remote("data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv", keep_local=True, static=True)
+    output: "data/load_raw.csv"
+    shell: "mv {input} {output}"
+
+
 rule build_load_data:
+    input: "data/load_raw.csv"
     output: "resources/load.csv"
     log: "logs/build_load_data.log"
     script: 'scripts/build_load_data.py'

config.default.yaml

@@ -188,7 +188,6 @@ transformers:
   type: ''
 
 load:
-  url: https://data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv
   power_statistics: True # only for files from <2019; set false in order to get ENTSOE transparency data 
   interpolate_limit: 3 # data gaps up until this size are interpolated linearly
   time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from 

config.tutorial.yaml

@@ -150,7 +150,6 @@ transformers:
   type: ''
 
 load:
-  url: https://data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv
   power_statistics: True # only for files from <2019; set false in order to get ENTSOE transparency data 
   interpolate_limit: 3 # data gaps up until this size are interpolated linearly
   time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from 

scripts/build_load_data.py

@@ -14,7 +14,6 @@ Relevant Settings
     snapshots:
 
     load:
-        url:
         interpolate_limit:
         time_shift_for_large_gaps:
         manual_adjustments:
@@ -199,14 +198,13 @@ if __name__ == "__main__":
 
     config = snakemake.config
     powerstatistics = config['load']['power_statistics']
-    url = config['load']['url']
     interpolate_limit = config['load']['interpolate_limit']
     countries = config['countries']
     snapshots = pd.date_range(freq='h', **config['snapshots'])
     years = slice(snapshots[0], snapshots[-1])
     time_shift = config['load']['time_shift_for_large_gaps']
 
-    load = load_timeseries(url, years, countries, powerstatistics)
+    load = load_timeseries(snakemake.input[0], years, countries, powerstatistics)
 
     if config['load']['manual_adjustments']:
         load = manual_adjustment(load, powerstatistics)

test/config.test1.yaml

@@ -149,7 +149,6 @@ transformers:
   type: ''
 
 load:
-  url: https://data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv
   power_statistics: True # only for files from <2019; set false in order to get ENTSOE transparency data
   interpolate_limit: 3 # data gaps up until this size are interpolated linearly
   time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from