From 9cd449cf3c24bc935b43c0bb1dcb3ff80015822e Mon Sep 17 00:00:00 2001
From: Fabian <fab.hof@gmx.de>
Date: Wed, 16 Aug 2023 11:41:48 +0200
Subject: [PATCH] retrieve electricity demand: use script in order to concat
 time-series

---
 rules/postprocess.smk                  |  1 -
 rules/retrieve.smk                     | 14 ++--------
 scripts/retrieve_electricity_demand.py | 37 ++++++++++++++++++++++++++
 3 files changed, 39 insertions(+), 13 deletions(-)
 create mode 100644 scripts/retrieve_electricity_demand.py

diff --git a/rules/postprocess.smk b/rules/postprocess.smk
index fccda6e4..4c833b4f 100644
--- a/rules/postprocess.smk
+++ b/rules/postprocess.smk
@@ -5,7 +5,6 @@
 
 localrules:
     copy_config,
-    copy_conda_env,
 
 
 rule plot_network:
diff --git a/rules/retrieve.smk b/rules/retrieve.smk
index 4b9e9542..d7e22e71 100644
--- a/rules/retrieve.smk
+++ b/rules/retrieve.smk
@@ -156,16 +156,6 @@ if config["enable"]["retrieve"] and (
 if config["enable"]["retrieve"]:
 
     rule retrieve_electricity_demand:
-        input:
-            HTTP.remote(
-                "data.open-power-system-data.org/time_series/{version}/time_series_60min_singleindex.csv".format(
-                version="2019-06-05"
-                    if config["snapshots"]["end"] < "2019"
-                    else "2020-10-06"
-                ),
-                keep_local=True,
-                static=True,
-            ),
         output:
             "data/load_raw.csv",
         log:
@@ -173,8 +163,8 @@ if config["enable"]["retrieve"]:
         resources:
             mem_mb=5000,
         retries: 2
-        run:
-            move(input[0], output[0])
+        script:
+            "../scripts/retrieve_electricity_demand.py"
 
 
 if config["enable"]["retrieve"]:
diff --git a/scripts/retrieve_electricity_demand.py b/scripts/retrieve_electricity_demand.py
new file mode 100644
index 00000000..58615755
--- /dev/null
+++ b/scripts/retrieve_electricity_demand.py
@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+"""
+Retrieve monthly fuel prices from Destatis.
+"""
+
+import logging
+
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+from pathlib import Path
+
+from _helpers import configure_logging, set_scenario_config
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake("retrieve_eletricity_demand")
+        rootpath = ".."
+    else:
+        rootpath = "."
+    configure_logging(snakemake)
+    set_scenario_config(snakemake)
+
+    versions = ["2019-06-05", "2020-10-06"]
+    url = "https://data.open-power-system-data.org/time_series/{version}/time_series_60min_singleindex.csv"
+
+    df1, df2 = [
+        pd.read_csv(url.format(version=version), index_col=0) for version in versions
+    ]
+    res = pd.concat([df1, df2[df2.index > df1.index[-1]]], join="inner")
+    res.to_csv(snakemake.output[0])