From c0d1656091d50eaf347967bdf26803a0b8fc6904 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 10 Nov 2021 10:37:09 +0100 Subject: [PATCH 1/3] build_load_data: separate retrieving and processing load data --- Snakefile | 7 +++++++ config.default.yaml | 1 - config.tutorial.yaml | 1 - scripts/build_load_data.py | 4 +--- test/config.test1.yaml | 1 - 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Snakefile b/Snakefile index 1196d47b..cb50e3bf 100644 --- a/Snakefile +++ b/Snakefile @@ -67,7 +67,14 @@ if config['enable'].get('retrieve_databundle', True): script: 'scripts/retrieve_databundle.py' +rule retrieve_load_data: + input: HTTP.remote("data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv", keep_local=True, static=True) + output: "data/load_raw.csv" + shell: "mv {input} {output}" + + rule build_load_data: + input: "data/load_raw.csv" output: "resources/load.csv" log: "logs/build_load_data.log" script: 'scripts/build_load_data.py' diff --git a/config.default.yaml b/config.default.yaml index a0ffbe47..f70e7c2c 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -188,7 +188,6 @@ transformers: type: '' load: - url: https://data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv power_statistics: True # only for files from <2019; set false in order to get ENTSOE transparency data interpolate_limit: 3 # data gaps up until this size are interpolated linearly time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from diff --git a/config.tutorial.yaml b/config.tutorial.yaml index c199712a..26ead242 100755 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -150,7 +150,6 @@ transformers: type: '' load: - url: https://data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv power_statistics: True # only for files from <2019; set false in order to get ENTSOE transparency data interpolate_limit: 3 # data gaps up until this size are interpolated linearly time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from diff --git a/scripts/build_load_data.py b/scripts/build_load_data.py index b286df45..f71be6ea 100755 --- a/scripts/build_load_data.py +++ b/scripts/build_load_data.py @@ -14,7 +14,6 @@ Relevant Settings snapshots: load: - url: interpolate_limit: time_shift_for_large_gaps: manual_adjustments: @@ -199,14 +198,13 @@ if __name__ == "__main__": config = snakemake.config powerstatistics = config['load']['power_statistics'] - url = config['load']['url'] interpolate_limit = config['load']['interpolate_limit'] countries = config['countries'] snapshots = pd.date_range(freq='h', **config['snapshots']) years = slice(snapshots[0], snapshots[-1]) time_shift = config['load']['time_shift_for_large_gaps'] - load = load_timeseries(url, years, countries, powerstatistics) + load = load_timeseries(snakemake.input[0], years, countries, powerstatistics) if config['load']['manual_adjustments']: load = manual_adjustment(load, powerstatistics) diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 6b3e667a..2986037b 100755 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -149,7 +149,6 @@ transformers: type: '' load: - url: https://data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv power_statistics: True # only for files from <2019; set false in order to get ENTSOE transparency data interpolate_limit: 3 # data gaps up until this size are interpolated linearly time_shift_for_large_gaps: 1w # data gaps up until this size are copied by copying from From f06c7958bc074ed295fdc773b39278a79fe8b450 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Sat, 4 Dec 2021 18:21:08 +0100 Subject: [PATCH 2/3] restore REUSE compliance [skip ci] --- .syncignore-receive | 4 ++++ .syncignore-send | 10 +++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.syncignore-receive b/.syncignore-receive index c24928f5..717245c3 100644 --- a/.syncignore-receive +++ b/.syncignore-receive @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: : 2021 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC0-1.0 + .snakemake .git .pytest_cache diff --git a/.syncignore-send b/.syncignore-send index 3839d915..9390d0e5 100644 --- a/.syncignore-send +++ b/.syncignore-send @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: : 2021 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC0-1.0 + .snakemake .git .pytest_cache @@ -11,9 +15,9 @@ __pycache__ notebooks benchmarks logs -resources +resources* results -networks +networks* cutouts data/bundle -doc \ No newline at end of file +doc From ed83988ed1fbc2b064cf5d2868965be9921bd12f Mon Sep 17 00:00:00 2001 From: Philipp Glaum Date: Wed, 15 Dec 2021 11:02:03 +0100 Subject: [PATCH 3/3] hot fix for snakemake bug --- scripts/_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 30775ae5..f1e5e887 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -240,7 +240,7 @@ def mock_snakemake(rulename, **wildcards): if os.path.exists(p): snakefile = p break - workflow = sm.Workflow(snakefile) + workflow = sm.Workflow(snakefile, overwrite_configfiles=[]) workflow.include(snakefile) workflow.global_resources = {} rule = workflow.get_rule(rulename)