add backward compatible config in wildcards

config/config.default.yaml

@@ -60,6 +60,14 @@ snapshots:
   end: "2014-01-01"
   inclusive: 'left'
 
+snapshot_opts:
+  average_every_nhours:
+    enable: false
+    hour: 2
+  time_segmentation:
+    enable: false
+    hour: 4380
+
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#enable
 enable:
   retrieve: auto
@@ -73,6 +81,22 @@ enable:
   retrieve_natura_raster: true
   custom_busmap: false
 
+enable_sector:
+  no_heat_district: false
+  land_transport: false
+  heating: false
+  waste_heat: false
+  biomass: false
+  biomass_transport: false
+  agriculture_machinery: false
+  industry: false
+  decentral: false
+  #wave_energy: false
+  #wave_energy_factor:
+  noH2network: false
+  carbon_budget: false
+  co2limit_sector: false
+
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#co2-budget
 co2_budget:
   2020: 0.701
@@ -83,10 +107,22 @@ co2_budget:
   2045: 0.032
   2050: 0.000
 
+co2_budget_opts:
+  from_descrete_value:
+    enable: true
+  from_beta_decay:
+    enable: false # TODO: move to own rule with sector-opts wildcard?
+    value: 1
+  from_exp_decay:
+    enable: false
+    value: 1
+
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#electricity
 electricity:
   voltages: [220., 300., 380.]
+  gaslimit_enable: false
   gaslimit: false
+  co2limit_enable: false
   co2limit: 7.75e+7
   co2base: 1.487e+9
   agg_p_nom_limits: data/agg_p_nom_minmax.csv
@@ -123,6 +159,17 @@ electricity:
       Onshore: [onwind]
       PV: [solar]
 
+  adjust_carrier: # This is the solar+c0.5 thing
+    enable: false
+    #solar:
+    #  p_nom_max:
+    #  capital_cost:
+    #  marginal_cost:
+
+  autarky:
+    enable: false
+    by_country: false
+
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#atlite
 atlite:
   default_cutout: europe-2013-era5
@@ -573,6 +620,11 @@ costs:
   emission_prices:
     co2: 0.
 
+  enable:
+    emission_prices: false
+    monthly_prices: false
+
+
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#clustering
 clustering:
   simplify_network:

rules/build_electricity.smk

@@ -473,10 +473,14 @@ rule prepare_network:
         links=config["links"],
         lines=config["lines"],
         co2base=config["electricity"]["co2base"],
+        co2limit_enable=config["electricity"].get("co2limit_enable", False),
         co2limit=config["electricity"]["co2limit"],
+        gaslimit_enable=config["electricity"].get("gaslimit_enable", False),
         gaslimit=config["electricity"].get("gaslimit"),
         max_hours=config["electricity"]["max_hours"],
         costs=config["costs"],
+        snapshot_opts=config.get("snapshot_opts",{}),
+        autarky=config["electricity"].get("autarky",{}),
     input:
         RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc",
         tech_costs=COSTS,

rules/build_sector.smk

@@ -705,6 +705,7 @@ rule build_transport_demand:
 
 rule prepare_sector_network:
     params:
+        enable_sector=config.get("enable_sector", {}),
         co2_budget=config["co2_budget"],
         conventional_carriers=config["existing_capacities"]["conventional_carriers"],
         foresight=config["foresight"],
@@ -717,6 +718,7 @@ rule prepare_sector_network:
         countries=config["countries"],
         emissions_scope=config["energy"]["emissions"],
         eurostat_report_year=config["energy"]["eurostat_report_year"],
+        snapshot_opts=config.get("snapshot_opts",{}),
         RDIR=RDIR,
     input:
         **build_retro_cost_output,

scripts/_helpers.py

@@ -7,6 +7,7 @@ import contextlib
 import logging
 import os
 import urllib
+import re
 from pathlib import Path
 
 import pandas as pd
@@ -21,6 +22,19 @@ logger = logging.getLogger(__name__)
 REGION_COLS = ["geometry", "name", "x", "y", "country"]
 
 
+def get_opt(opts, expr, flags=None):
+    """
+    Return the first option matching the regular expression.
+    The regular expression is case-insensitive by default.
+    """
+    if flags is None:
+        flags = re.IGNORECASE
+    for o in opts:
+        match = re.match(expr, o, flags=flags)
+        if match:
+            return match.group(0)
+    return None
+
 # Define a context manager to temporarily mute print statements
 @contextlib.contextmanager
 def mute_print():

scripts/prepare_network.py

@@ -63,7 +63,7 @@ import re
 import numpy as np
 import pandas as pd
 import pypsa
-from _helpers import configure_logging
+from _helpers import configure_logging, get_opt
 from add_electricity import load_costs, update_transmission_costs
 from pypsa.descriptors import expand_series
 
@@ -71,6 +71,18 @@ idx = pd.IndexSlice
 
 logger = logging.getLogger(__name__)
 
+def find_opt(opts, expr):
+    """
+    Return if available the float after the expression.
+    """
+    for o in opts:
+        if expr in o:
+            m = re.findall("[0-9]*\.?[0-9]+$", o)
+            if len(m) > 0:
+                return True, float(m[0])
+            else:
+                return True, None
+    return False, None
 
 def add_co2limit(n, co2limit, Nyears=1.0):
     n.add(
@@ -297,42 +309,46 @@ if __name__ == "__main__":
 
     set_line_s_max_pu(n, snakemake.params.lines["s_max_pu"])
     
-    for o in opts:
+    # temporal averaging
-        m = re.match(r"^\d+h$", o, re.IGNORECASE)
+    nhours_opts_config = snakemake.params.snapshot_opts.get("average_every_nhours",{})
-        if m is not None:
+    nhours_enable_config = nhours_opts_config.get("enable",None)
-            n = average_every_nhours(n, m.group(0))
+    nhours_config = str(nhours_opts_config.get("hour",None)) + "h"
-            break
+    nhours_wildcard = get_opt(opts, r"^\d+h$")
+    if nhours_wildcard is not None or (nhours_enable_config and nhours_config is not None):
+        nhours = nhours_wildcard or nhours_config
+        n = average_every_nhours(n, nhours)
 
-    for o in opts:
+    # segments with package tsam
-        m = re.match(r"^\d+seg$", o, re.IGNORECASE)
+    time_seg_opts_config = snakemake.params.snapshot_opts.get("time_segmentation",{})
-        if m is not None:
+    time_seg_enable_config = nhours_opts_config.get("enable",None)
+    time_seg_config = str(nhours_opts_config.get("hour",None)) + "seg"
+    time_seg_wildcard = get_opt(opts, r"^\d+seg$")
+    if time_seg_wildcard is not None or (time_seg_enable_config and time_seg_config is not None):
+        time_seg = time_seg_wildcard or time_seg_config
         solver_name = snakemake.config["solving"]["solver"]["name"]
-            n = apply_time_segmentation(n, m.group(0)[:-3], solver_name)
+        n = apply_time_segmentation(n, time_seg, solver_name)
-            break
 
-    for o in opts:
+    Co2L_config = snakemake.params.co2limit_enable and isinstance(snakemake.params.co2limit,float)
-        if "Co2L" in o:
+    Co2L_wildcard, co2limit_wildcard = find_opt(opts, "Co2L")
-            m = re.findall("[0-9]*\.?[0-9]+$", o)
+    if Co2L_wildcard or Co2L_config:
-            if len(m) > 0:
+        if co2limit_wildcard is not None: # TODO: what if you wat to determine the factor through the wildcard?
-                co2limit = float(m[0]) * snakemake.params.co2base
+            co2limit = co2limit_wildcard * snakemake.params.co2base
             add_co2limit(n, co2limit, Nyears)
             logger.info("Setting CO2 limit according to wildcard value.")
         else: 
             add_co2limit(n, snakemake.params.co2limit, Nyears)
             logger.info("Setting CO2 limit according to config value.")
-            break
 
-    for o in opts:
+    CH4L_config = snakemake.params.gaslimit_enable and isinstance(snakemake.params.gaslimit,float)
-        if "CH4L" in o:
+    CH4L_wildcard, gaslimit_wildcard = find_opt(opts, "CH4L")
-            m = re.findall("[0-9]*\.?[0-9]+$", o)
+    if CH4L_wildcard or CH4L_config:
-            if len(m) > 0:
+        if gaslimit_wildcard is not None: # TODO: what if you wat to determine the factor through the wildcard?
-                limit = float(m[0]) * 1e6
+            gaslimit = gaslimit_wildcard * 1e6
-                add_gaslimit(n, limit, Nyears)
+            add_gaslimit(n, gaslimit, Nyears)
             logger.info("Setting gas usage limit according to wildcard value.")
         else: 
             add_gaslimit(n, snakemake.params.gaslimit, Nyears)
             logger.info("Setting gas usage limit according to config value.")
-            break
 
     for o in opts:
         if "+" not in o:
@@ -353,21 +369,24 @@ if __name__ == "__main__":
                     sel = c.df.carrier.str.contains(carrier)
                     c.df.loc[sel, attr] *= factor
 
+    Ept_config = snakemake.params.costs.get("enable",{}).get("monthly_prices", False)
     for o in opts:
-        if "Ept" in o:
+        if "Ept" in o or Ept_config:
             logger.info(
                 "Setting time dependent emission prices according spot market price"
             )
             add_dynamic_emission_prices(n)
-        elif "Ep" in o:
+            Ept_config = True
-            m = re.findall("[0-9]*\.?[0-9]+$", o)
+
-            if len(m) > 0:
+    Ep_config = snakemake.params.costs.get("enable",{}).get("emission_prices", False)
+    Ep_wildcard, co2_wildcard = find_opt(opts, "Ep")
+    if (Ep_wildcard or Ep_config) and not Ept_config:
+        if co2_wildcard is not None:
             logger.info("Setting emission prices according to wildcard value.")
-                add_emission_prices(n, dict(co2=float(m[0])))
+            add_emission_prices(n, dict(co2=co2_wildcard))
         else: 
             logger.info("Setting emission prices according to config value.")
             add_emission_prices(n, snakemake.params.costs["emission_prices"])
-            break
 
     ll_type, factor = snakemake.wildcards.ll[0], snakemake.wildcards.ll[1:]
     set_transmission_limit(n, ll_type, factor, costs, Nyears)
@@ -380,10 +399,12 @@ if __name__ == "__main__":
         p_nom_max_ext=snakemake.params.links.get("max_extension", np.inf),
     )
 
-    if "ATK" in opts:
+    autarky_config = snakemake.params.autarky
-        enforce_autarky(n)
+    if "ATK" in opts or autarky_config.get("enable", False):
-    elif "ATKc" in opts:
+        only_crossborder = False
-        enforce_autarky(n, only_crossborder=True)
+        if "ATKc" in opts or autarky_config.get("by_country", False):
+            only_crossborder = True
+        enforce_autarky(n, only_crossborder=only_crossborder)
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
     n.export_to_netcdf(snakemake.output[0])

scripts/prepare_sector_network.py

@@ -17,7 +17,7 @@ import numpy as np
 import pandas as pd
 import pypsa
 import xarray as xr
-from _helpers import generate_periodic_profiles, update_config_with_sector_opts
+from _helpers import generate_periodic_profiles, update_config_with_sector_opts, get_opt
 from add_electricity import calculate_annuity, sanitize_carriers
 from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2
 from networkx.algorithms import complement
@@ -161,11 +161,11 @@ spatial = SimpleNamespace()
 
 def emission_sectors_from_opts(opts):
     sectors = ["electricity"]
-    if "T" in opts:
+    if "T" in opts or opts_config.get("land_transport",False):
         sectors += ["rail non-elec", "road non-elec"]
-    if "H" in opts:
+    if "H" in opts or opts_config.get("heating",False):
         sectors += ["residential non-elec", "services non-elec"]
-    if "I" in opts:
+    if "I" in opts or opts_config.get("industry",False):
         sectors += [
             "industrial non-elec",
             "industrial processes",
@@ -174,7 +174,10 @@ def emission_sectors_from_opts(opts):
             "domestic navigation",
             "international navigation",
         ]
-    if "A" in opts:
+
+    heat_and_industry = opts_config.get("industry",False) and opts_config.get("heating",False)
+
+    if ("I" in opts and "H" in opts and"A" in opts) or (heat_and_industry and opts_config.get("agriculture_machinery",False)):
         sectors += ["agriculture"]
 
     return sectors
@@ -3256,27 +3259,39 @@ def set_temporal_aggregation(n, opts, solver_name):
     """
     Aggregate network temporally.
     """
-    for o in opts:
     # temporal averaging
-        m = re.match(r"^\d+h$", o, re.IGNORECASE)
+    nhours_opts_config = snakemake.params.snapshot_opts.get("average_every_nhours",{})
-        if m is not None:
+    nhours_enable_config = nhours_opts_config.get("enable",None)
-            n = average_every_nhours(n, m.group(0))
+    nhours_config = str(nhours_opts_config.get("hour",None)) + "H"
-            break
+    nhours_wildcard = get_opt(opts, r"^\d+h$")
+    if nhours_wildcard is not None or (nhours_enable_config and nhours_config is not None):
+        nhours = nhours_wildcard or nhours_config
+        n = average_every_nhours(n, nhours)
+        return n
+
     # representative snapshots
-        m = re.match(r"(^\d+)sn$", o, re.IGNORECASE)
+    snapshots_opts_config = snakemake.params.snapshot_opts.get("set_snapshots",{})
-        if m is not None:
+    snapshots_enable_config = snapshots_opts_config.get("enable",None)
-            sn = int(m[1])
+    snapshots_config = snapshots_opts_config.get("hour",None)
+    snapshots_wildcard = get_opt(opts, r"(^\d+)sn$")
+    if snapshots_wildcard is not None or (snapshots_enable_config and snapshots_config is not None):
+        sn = int(snapshots_wildcard[:-2]) or snapshots_config
         logger.info(f"Use every {sn} snapshot as representative")
         n.set_snapshots(n.snapshots[::sn])
         n.snapshot_weightings *= sn
-            break
+        return n
+
     # segments with package tsam
-        m = re.match(r"^(\d+)seg$", o, re.IGNORECASE)
+    time_seg_opts_config = snakemake.params.snapshot_opts.get("time_segmentation",{})
-        if m is not None:
+    time_seg_enable_config = nhours_opts_config.get("enable",None)
-            segments = int(m[1])
+    time_seg_config = nhours_opts_config.get("hour",None)
+    time_seg_wildcard = get_opt(opts, r"^(\d+)seg$")
+    if time_seg_wildcard is not None or (time_seg_enable_config and time_seg_config is not None):
+        segments = int(time_seg_wildcard[:-3]) or time_seg_config
         logger.info(f"Use temporal segmentation with {segments} segments")
         n = apply_time_segmentation(n, segments, solver_name=solver_name)
-            break
+        return n
+
     return n
 
 
@@ -3303,6 +3318,10 @@ if __name__ == "__main__":
 
     opts = snakemake.wildcards.sector_opts.split("-")
 
+    opts_config = snakemake.params.enable_sector
+
+    heat_and_industry = opts_config.get("industry",False) and opts_config.get("heating",False)
+
     investment_year = int(snakemake.wildcards.planning_horizons[-4:])
 
     n = pypsa.Network(snakemake.input.network)
@@ -3340,51 +3359,53 @@ if __name__ == "__main__":
 
     # TODO merge with opts cost adjustment below
     for o in opts:
-        if o[:4] == "wave":
+        if o[:4] == "wave": # TODO: add config wildcard options or depreciated?
             wave_cost_factor = float(o[4:].replace("p", ".").replace("m", "-"))
             logger.info(
                 f"Including wave generators with cost factor of {wave_cost_factor}"
             )
             add_wave(n, wave_cost_factor)
-        if o[:4] == "dist":
+        if o[:4] == "dist": # TODO: add config wildcard options
             options["electricity_distribution_grid"] = True
             options["electricity_distribution_grid_cost_factor"] = float(
                 o[4:].replace("p", ".").replace("m", "-")
             )
-        if o == "biomasstransport":
+    for o in opts:
+        if o == "biomasstransport" or opts_config.get("biomass_transport",False):
             options["biomass_transport"] = True
+            break
 
-    if "nodistrict" in opts:
+    if "nodistrict" in opts or opts_config.get("no_heat_district",False):
         options["district_heating"]["progress"] = 0.0
 
-    if "T" in opts:
+    if "T" in opts or opts_config.get("land_transport",False):
         add_land_transport(n, costs)
 
-    if "H" in opts:
+    if "H" in opts or opts_config.get("heating",False):
         add_heat(n, costs)
 
-    if "B" in opts:
+    if "B" in opts or opts_config.get("biomass",False):
         add_biomass(n, costs)
 
     if options["ammonia"]:
         add_ammonia(n, costs)
 
-    if "I" in opts:
+    if "I" in opts or opts_config.get("industry",False):
         add_industry(n, costs)
 
-    if "I" in opts and "H" in opts:
+    if ("I" in opts and "H" in opts) or (heat_and_industry and opts_config.get("waste_heat",False)):
         add_waste_heat(n)
 
-    if "A" in opts:  # requires H and I
+    if ("I" in opts and "H" in opts and"A" in opts) or (heat_and_industry and opts_config.get("agriculture_machinery",False)):  # requires H and I
         add_agriculture(n, costs)
 
     if options["dac"]:
         add_dac(n, costs)
 
-    if "decentral" in opts:
+    if "decentral" in opts or opts_config.get("decentral",False):
         decentral(n)
 
-    if "noH2network" in opts:
+    if "noH2network" in opts or opts_config.get("noH2network",False):
         remove_h2_network(n)
 
     if options["co2network"]:
@@ -3399,7 +3420,7 @@ if __name__ == "__main__":
     limit_type = "config"
     limit = get(snakemake.params.co2_budget, investment_year)
     for o in opts:
-        if "cb" not in o:
+        if "cb" not in o or opts_config.get("carbon_budget",False) is False:
             continue
         limit_type = "carbon budget"
         fn = "results/" + snakemake.params.RDIR + "csvs/carbon_budget_distribution.csv"
@@ -3419,7 +3440,7 @@ if __name__ == "__main__":
         limit = co2_cap.loc[investment_year]
         break
     for o in opts:
-        if "Co2L" not in o:
+        if "Co2L" not in o or opts_config.get("co2limit_sector",False) is False:
             continue
         limit_type = "wildcard"
         limit = o[o.find("Co2L") + 4 :]
@@ -3428,7 +3449,7 @@ if __name__ == "__main__":
     logger.info(f"Add CO2 limit from {limit_type}")
     add_co2limit(n, nyears, limit)
 
-    for o in opts:
+    for o in opts: # TODO: add config wildcard options or depreciated?
         if not o[:10] == "linemaxext":
             continue
         maxext = float(o[10:]) * 1e3