From fe143ff8fef243d0fb78e420ee7a8274aac02b39 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 12 Feb 2024 12:10:05 +0100 Subject: [PATCH 1/6] use default config as base configuration --- Snakefile | 11 +++-------- config/config.yaml | 7 +++++++ doc/configuration.rst | 2 +- doc/installation.rst | 13 ++++--------- doc/introduction.rst | 2 +- doc/release_notes.rst | 5 +++++ 6 files changed, 21 insertions(+), 19 deletions(-) create mode 100644 config/config.yaml diff --git a/Snakefile b/Snakefile index 8391156c..ba60cbc9 100644 --- a/Snakefile +++ b/Snakefile @@ -2,8 +2,8 @@ # # SPDX-License-Identifier: MIT -from os.path import normpath, exists -from shutil import copyfile, move, rmtree +from os.path import normpath +from shutil import move, rmtree from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider @@ -13,12 +13,7 @@ from snakemake.utils import min_version min_version("7.7") -conf_file = os.path.join(workflow.current_basedir, "config/config.yaml") -conf_default_file = os.path.join(workflow.current_basedir, "config/config.default.yaml") -if not exists(conf_file) and exists(conf_default_file): - copyfile(conf_default_file, conf_file) - - +configfile: "config/config.default.yaml" configfile: "config/config.yaml" diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 00000000..6fa1862c --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: : 2024 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC0-1.0 +# add your own configuration overrides here, for instance + +# enable: +# retrieve: false \ No newline at end of file diff --git a/doc/configuration.rst b/doc/configuration.rst index 92ed269b..f65aa4c2 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -9,7 +9,7 @@ Configuration ########################################## -PyPSA-Eur has several configuration options which are documented in this section and are collected in a ``config/config.yaml`` file located in the root directory. Users should copy the provided default configuration (``config/config.default.yaml``) and amend their own modifications and assumptions in the user-specific configuration file (``config/config.yaml``); confer installation instructions at :ref:`defaultconfig`. +PyPSA-Eur has several configuration options which are documented in this section and are collected in a ``config/config.yaml`` file. This file defines deviations from the default configuration (``config/config.default.yaml``); confer installation instructions at :ref:`defaultconfig`. .. _toplevel_cf: diff --git a/doc/installation.rst b/doc/installation.rst index 01fdafeb..d2fdfb4e 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -118,11 +118,10 @@ Nevertheless, you can still use open-source solvers for smaller problems. Handling Configuration Files ============================ -PyPSA-Eur has several configuration options that must be specified in a -``config/config.yaml`` file located in the root directory. An example configuration -``config/config.default.yaml`` is maintained in the repository, which will be used to -automatically create your customisable ``config/config.yaml`` on first use. More -details on the configuration options are in :ref:`config`. +PyPSA-Eur has several configuration options that users can specify in a +``config/config.yaml`` file. The default configuration +``config/config.default.yaml`` is maintained in the repository. More details on +the configuration options are in :ref:`config`. You can also use ``snakemake`` to specify another file, e.g. ``config/config.mymodifications.yaml``, to update the settings of the ``config/config.yaml``. @@ -131,7 +130,3 @@ You can also use ``snakemake`` to specify another file, e.g. .../pypsa-eur % snakemake -call --configfile config/config.mymodifications.yaml -.. warning:: - Users are advised to regularly check their own ``config/config.yaml`` against changes - in the ``config/config.default.yaml`` when pulling a new version from the remote - repository. diff --git a/doc/introduction.rst b/doc/introduction.rst index 413db9d1..0047f0a7 100644 --- a/doc/introduction.rst +++ b/doc/introduction.rst @@ -74,7 +74,7 @@ what data to retrieve and what files to produce. Details are explained in :ref:`wildcards` and :ref:`scenario`. The model also has several further configuration options collected in the -``config/config.yaml`` file located in the root directory, which that are not part of +``config/config.default.yaml`` file located in the root directory, which that are not part of the scenarios. Options are explained in :ref:`config`. Folder Structure diff --git a/doc/release_notes.rst b/doc/release_notes.rst index fce4ae1b..cab8229a 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,11 @@ Release Notes Upcoming Release ================ +* The default configuration ``config/config.default.yaml`` is now automatically + used as a base configuration file and no longer copied to + ``config/config.yaml`` on first use. The file ``config/config.yaml`` should be + used to define deviations from the default configuration. + * Merged two OPSD time series data versions into such that the option ``load: power_statistics:`` becomes superfluous and was hence removed. From abe06a68fbc28c3d71c459eb66ed7daf5b8d42b0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 12 Feb 2024 11:11:02 +0000 Subject: [PATCH 2/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Snakefile | 1 + config/config.yaml | 2 +- doc/installation.rst | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Snakefile b/Snakefile index ba60cbc9..ee24841a 100644 --- a/Snakefile +++ b/Snakefile @@ -13,6 +13,7 @@ from snakemake.utils import min_version min_version("7.7") + configfile: "config/config.default.yaml" configfile: "config/config.yaml" diff --git a/config/config.yaml b/config/config.yaml index 6fa1862c..6b8bdd77 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -4,4 +4,4 @@ # add your own configuration overrides here, for instance # enable: -# retrieve: false \ No newline at end of file +# retrieve: false diff --git a/doc/installation.rst b/doc/installation.rst index d2fdfb4e..4289b098 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -129,4 +129,3 @@ You can also use ``snakemake`` to specify another file, e.g. .. code:: bash .../pypsa-eur % snakemake -call --configfile config/config.mymodifications.yaml - From 5840a9aa11378a028504e09edd91322c53b0b2cb Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 12 Feb 2024 12:17:13 +0100 Subject: [PATCH 3/6] remove inefficient _helper.py file search --- rules/common.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/common.smk b/rules/common.smk index 5aa7ae53..4c6cc7c2 100644 --- a/rules/common.smk +++ b/rules/common.smk @@ -4,7 +4,7 @@ import os, sys, glob -helper_source_path = [match for match in glob.glob("**/_helpers.py", recursive=True)] +helper_source_path = workflow.source_path("scripts/_helpers.py") for path in helper_source_path: path = os.path.dirname(os.path.abspath(path)) From 1e5163b66bc5cd3d67f3b7b84b7997ebf2d3a5be Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 12 Feb 2024 13:45:42 +0100 Subject: [PATCH 4/6] alternative approach --- rules/common.smk | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/rules/common.smk b/rules/common.smk index 4c6cc7c2..50132f4b 100644 --- a/rules/common.smk +++ b/rules/common.smk @@ -4,11 +4,8 @@ import os, sys, glob -helper_source_path = workflow.source_path("scripts/_helpers.py") - -for path in helper_source_path: - path = os.path.dirname(os.path.abspath(path)) - sys.path.insert(0, os.path.abspath(path)) +for path in ["../scripts", "./scripts"]: + sys.path.insert(0, os.path.abspath(path) from _helpers import validate_checksum From fa931cc2b102ebd8a7241e0e0aba8bd8281e1eb5 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 12 Feb 2024 13:54:28 +0100 Subject: [PATCH 5/6] alternative approach 2 --- rules/common.smk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rules/common.smk b/rules/common.smk index 50132f4b..598d3b98 100644 --- a/rules/common.smk +++ b/rules/common.smk @@ -4,8 +4,8 @@ import os, sys, glob -for path in ["../scripts", "./scripts"]: - sys.path.insert(0, os.path.abspath(path) +path = workflow.source_path("../scripts/_helpers.py") +sys.path.insert(0, os.path.dirname(path)) from _helpers import validate_checksum From 69e20e344864245039bb392089b9f2645799ba2a Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 12 Feb 2024 13:55:32 +0100 Subject: [PATCH 6/6] need at least one entry for config/config.yaml to read as yaml --- config/config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/config.yaml b/config/config.yaml index 6fa1862c..620a695c 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -3,5 +3,6 @@ # SPDX-License-Identifier: CC0-1.0 # add your own configuration overrides here, for instance +version: 0.9.0 # enable: # retrieve: false \ No newline at end of file