Merge branch 'master' into multiyear

doc/release_notes.rst

@@ -56,6 +56,9 @@ Upcoming Release
     script can be found in this `Github gist
     <https://gist.github.com/fneum/47b857862dd9148a22eca5a2e85caa9a>`_.
 
+* Removed rule ``copy_config``. Instead, a config file is created for each
+  network output of the ``solve_*`` rules, with the same content as ``n.meta``.
+
 * Upgrade to Snakemake v8.5+. This version is the new minimum version required.
   To upgrade an existing environment, run ``conda install -c bioconda
   snakemake-minimal">=8.5"`` and ``pip install snakemake-storage-plugin-http``
@@ -152,6 +155,8 @@ Upcoming Release
 
 * Adapt the disabling of transmission expansion in myopic foresight optimisations when limit is already reached to also handle cost limits.
 
+* Fix duplicated years in `add_land_use_constraint_m`.
+
 PyPSA-Eur 0.10.0 (19th February 2024)
 =====================================
 

doc/sector.rst

@@ -176,11 +176,6 @@ Rule ``cluster_gas_network``
 
 .. automodule:: cluster_gas_network
 
-Rule ``copy_config``
-==============================================================================
-
-.. automodule:: copy_config
-
 Rule ``prepare_sector_network``
 ==============================================================================
 

rules/postprocess.smk

@@ -3,10 +3,6 @@
 # SPDX-License-Identifier: MIT
 
 
-localrules:
-    copy_config,
-
-
 if config["foresight"] != "perfect":
 
     rule plot_power_network_clustered:
@@ -137,20 +133,6 @@ if config["foresight"] == "perfect":
             "../scripts/plot_power_network_perfect.py"
 
 
-rule copy_config:
-    params:
-        RDIR=RDIR,
-    output:
-        RESULTS + "config.yaml",
-    threads: 1
-    resources:
-        mem_mb=1000,
-    conda:
-        "../envs/environment.yaml"
-    script:
-        "../scripts/copy_config.py"
-
-
 rule make_summary:
     params:
         foresight=config_provider("foresight"),

rules/solve_electricity.smk

@@ -14,9 +14,9 @@ rule solve_network:
         custom_extra_functionality=input_custom_extra_functionality,
     input:
         network=resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"),
-        config=RESULTS + "config.yaml",
     output:
         network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
+        config=RESULTS + "configs/config.elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.yaml",
     log:
         solver=normpath(
             RESULTS

rules/solve_myopic.smk

@@ -118,10 +118,11 @@ rule solve_sector_network_myopic:
         network=RESULTS
         + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
         costs=resources("costs_{planning_horizons}.csv"),
-        config=RESULTS + "config.yaml",
     output:
-        RESULTS
+        network=RESULTS
         + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
+        config=RESULTS
+        + "configs/config.elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.yaml",
     shadow:
         "shallow"
     log:

rules/solve_overnight.smk

@@ -15,10 +15,11 @@ rule solve_sector_network:
     input:
         network=RESULTS
         + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
-        config=RESULTS + "config.yaml",
     output:
-        RESULTS
+        network=RESULTS
         + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
+        config=RESULTS
+        + "configs/config.elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.yaml",
     shadow:
         "shallow"
     log:

rules/solve_perfect.smk

@@ -107,10 +107,10 @@ rule solve_sector_network_perfect:
         network=RESULTS
         + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
         costs=resources("costs_2030.csv"),
-        config=RESULTS + "config.yaml",
     output:
-        RESULTS
+        network=RESULTS
         + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
+        config="configs/config.elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.yaml",
     threads: solver_threads
     resources:
         mem_mb=config_provider("solving", "mem"),

scripts/copy_config.py

@@ -1,28 +0,0 @@
-# -*- coding: utf-8 -*-
-# SPDX-FileCopyrightText: : 2020-2024 The PyPSA-Eur Authors
-#
-# SPDX-License-Identifier: MIT
-"""
-Copy used configuration files and important scripts for archiving.
-"""
-
-
-import yaml
-from _helpers import set_scenario_config
-
-if __name__ == "__main__":
-    if "snakemake" not in globals():
-        from _helpers import mock_snakemake
-
-        snakemake = mock_snakemake("copy_config")
-
-    set_scenario_config(snakemake)
-
-    with open(snakemake.output[0], "w") as yaml_file:
-        yaml.dump(
-            snakemake.config,
-            yaml_file,
-            default_flow_style=False,
-            allow_unicode=True,
-            sort_keys=False,
-        )

scripts/solve_network.py

@@ -36,6 +36,7 @@ import numpy as np
 import pandas as pd
 import pypsa
 import xarray as xr
+import yaml
 from _benchmark import memory_logger
 from _helpers import (
     configure_logging,
@@ -165,7 +166,7 @@ def _add_land_use_constraint_m(n, planning_horizons, config):
 
         previous_years = [
             str(y)
-            for y in planning_horizons + grouping_years
+            for y in set(planning_horizons + grouping_years)
             if y < int(snakemake.wildcards.planning_horizons)
         ]
 
@@ -968,4 +969,13 @@ if __name__ == "__main__":
     logger.info(f"Maximum memory usage: {mem.mem_usage}")
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
-    n.export_to_netcdf(snakemake.output[0])
+    n.export_to_netcdf(snakemake.output.network)
+
+    with open(snakemake.output.config, "w") as file:
+        yaml.dump(
+            n.meta,
+            file,
+            default_flow_style=False,
+            allow_unicode=True,
+            sort_keys=False,
+        )