From cb00ba58ceb2241ff6a12ea7d623d6fb781c9c57 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Wed, 20 Jul 2022 11:35:12 +0200
Subject: [PATCH 1/3] add option to specify any config through sector opts with
 CF:<label>:...:value

---
 scripts/add_brownfield.py         |  4 +++-
 scripts/add_existing_baseyear.py  |  4 +++-
 scripts/helper.py                 | 18 +++++++++++++++++-
 scripts/plot_summary.py           |  5 ++++-
 scripts/prepare_sector_network.py |  4 +++-
 scripts/solve_network.py          |  4 +++-
 6 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py
index d7418a79..9d2d86ce 100644
--- a/scripts/add_brownfield.py
+++ b/scripts/add_brownfield.py
@@ -11,7 +11,7 @@ import yaml
 import numpy as np
 
 from add_existing_baseyear import add_build_year_to_new_assets
-from helper import override_component_attrs
+from helper import override_component_attrs, update_config_with_sector_opts
 from solve_network import basename
 
 
@@ -123,6 +123,8 @@ if __name__ == "__main__":
             planning_horizons=2030,
         )
 
+    update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts)
+
     print(snakemake.input.network_p)
     logging.basicConfig(level=snakemake.config['logging_level'])
 
diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py
index 7bb88a00..11b8d49b 100644
--- a/scripts/add_existing_baseyear.py
+++ b/scripts/add_existing_baseyear.py
@@ -13,7 +13,7 @@ import pypsa
 import yaml
 
 from prepare_sector_network import prepare_costs, define_spatial
-from helper import override_component_attrs
+from helper import override_component_attrs, update_config_with_sector_opts
 
 from types import SimpleNamespace
 spatial = SimpleNamespace()
@@ -463,6 +463,8 @@ if __name__ == "__main__":
 
     logging.basicConfig(level=snakemake.config['logging_level'])
 
+    update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts)
+
     options = snakemake.config["sector"]
     opts = snakemake.wildcards.sector_opts.split('-')
 
diff --git a/scripts/helper.py b/scripts/helper.py
index 7144171c..51b97f29 100644
--- a/scripts/helper.py
+++ b/scripts/helper.py
@@ -1,7 +1,9 @@
 import os
+import yaml
 import pytz
 import pandas as pd
 from pathlib import Path
+from mergedeep import merge
 from pypsa.descriptors import Dict
 from pypsa.components import components, component_attrs
 
@@ -124,4 +126,18 @@ def generate_periodic_profiles(dt_index, nodes, weekly_profile, localize=None):
 
     week_df = week_df.tz_localize(localize)
 
-    return week_df
\ No newline at end of file
+    return week_df
+
+
+def parse(l):
+    if len(l) == 1:
+        return yaml.safe_load(l[0])
+    else:
+        return {l.pop(0): parse(l)}
+
+
+def update_config_with_sector_opts(config, sector_opts):
+    for o in sector_opts.split("-"):
+        if o.startswith("CF:"):
+            l = o.split(":")[1:]
+            merge(config, parse(l))
\ No newline at end of file
diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py
index 8b073b17..7ebfde24 100644
--- a/scripts/plot_summary.py
+++ b/scripts/plot_summary.py
@@ -7,6 +7,7 @@ import matplotlib.pyplot as plt
 plt.style.use('ggplot')
 
 from prepare_sector_network import co2_emissions_year
+from helper import update_config_with_sector_opts
 
 #consolidate and rename
 def rename_techs(label):
@@ -437,7 +438,9 @@ if __name__ == "__main__":
     if 'snakemake' not in globals():
         from helper import mock_snakemake
         snakemake = mock_snakemake('plot_summary')
-        
+    
+    update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts)
+
     n_header = 4
 
     plot_costs()
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 49eef826..7abdadff 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -14,7 +14,7 @@ from scipy.stats import beta
 from vresutils.costdata import annuity
 
 from build_energy_totals import build_eea_co2, build_eurostat_co2, build_co2_totals
-from helper import override_component_attrs, generate_periodic_profiles
+from helper import override_component_attrs, generate_periodic_profiles, update_config_with_sector_opts
 
 from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation
 from networkx.algorithms import complement
@@ -2339,6 +2339,8 @@ if __name__ == "__main__":
 
     logging.basicConfig(level=snakemake.config['logging_level'])
 
+    update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts)
+
     options = snakemake.config["sector"]
 
     opts = snakemake.wildcards.sector_opts.split('-')
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index 4f6cc2c4..b1c02500 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -11,7 +11,7 @@ from pypsa.linopf import network_lopf, ilopf
 
 from vresutils.benchmark import memory_logger
 
-from helper import override_component_attrs
+from helper import override_component_attrs, update_config_with_sector_opts
 
 import logging
 logger = logging.getLogger(__name__)
@@ -290,6 +290,8 @@ if __name__ == "__main__":
     logging.basicConfig(filename=snakemake.log.python,
                         level=snakemake.config['logging_level'])
 
+    update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts)
+
     tmpdir = snakemake.config['solving'].get('tmpdir')
     if tmpdir is not None:
         from pathlib import Path

From a65dd4e6c24c560e94e63f75868c053bcaa53c2f Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 24 Jul 2022 21:26:19 +0200
Subject: [PATCH 2/3] use snakemake.utils.update_config instead of mergedeep

---
 scripts/helper.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/helper.py b/scripts/helper.py
index 51b97f29..2d6bcc5d 100644
--- a/scripts/helper.py
+++ b/scripts/helper.py
@@ -3,7 +3,7 @@ import yaml
 import pytz
 import pandas as pd
 from pathlib import Path
-from mergedeep import merge
+from snakemake.utils import update_config
 from pypsa.descriptors import Dict
 from pypsa.components import components, component_attrs
 
@@ -140,4 +140,4 @@ def update_config_with_sector_opts(config, sector_opts):
     for o in sector_opts.split("-"):
         if o.startswith("CF:"):
             l = o.split(":")[1:]
-            merge(config, parse(l))
\ No newline at end of file
+            update_config(config, parse(l))
\ No newline at end of file

From 503ecd290735b221f5e65830b5454c94bf3dbc0c Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 24 Jul 2022 21:27:12 +0200
Subject: [PATCH 3/3] use + separator instead of :

---
 scripts/helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/helper.py b/scripts/helper.py
index 2d6bcc5d..48e28e71 100644
--- a/scripts/helper.py
+++ b/scripts/helper.py
@@ -139,5 +139,5 @@ def parse(l):
 def update_config_with_sector_opts(config, sector_opts):
     for o in sector_opts.split("-"):
         if o.startswith("CF:"):
-            l = o.split(":")[1:]
+            l = o.split("+")[1:]
             update_config(config, parse(l))
\ No newline at end of file