From 823df52309d1a1a44bb449802c2fdbaee8cb0d95 Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Mon, 11 Sep 2023 22:51:31 +0200 Subject: [PATCH] add backward compatible config in wildcards --- config/config.default.yaml | 52 +++++++++++++ rules/build_electricity.smk | 4 + rules/build_sector.smk | 2 + scripts/_helpers.py | 14 ++++ scripts/prepare_network.py | 119 ++++++++++++++++++------------ scripts/prepare_sector_network.py | 103 ++++++++++++++++---------- 6 files changed, 204 insertions(+), 90 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index b162b75d..fcc74981 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -60,6 +60,14 @@ snapshots: end: "2014-01-01" inclusive: 'left' +snapshot_opts: + average_every_nhours: + enable: false + hour: 2 + time_segmentation: + enable: false + hour: 4380 + # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#enable enable: retrieve: auto @@ -73,6 +81,22 @@ enable: retrieve_natura_raster: true custom_busmap: false +enable_sector: + no_heat_district: false + land_transport: false + heating: false + waste_heat: false + biomass: false + biomass_transport: false + agriculture_machinery: false + industry: false + decentral: false + #wave_energy: false + #wave_energy_factor: + noH2network: false + carbon_budget: false + co2limit_sector: false + # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#co2-budget co2_budget: 2020: 0.701 @@ -83,10 +107,22 @@ co2_budget: 2045: 0.032 2050: 0.000 +co2_budget_opts: + from_descrete_value: + enable: true + from_beta_decay: + enable: false # TODO: move to own rule with sector-opts wildcard? + value: 1 + from_exp_decay: + enable: false + value: 1 + # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#electricity electricity: voltages: [220., 300., 380.] + gaslimit_enable: false gaslimit: false + co2limit_enable: false co2limit: 7.75e+7 co2base: 1.487e+9 agg_p_nom_limits: data/agg_p_nom_minmax.csv @@ -123,6 +159,17 @@ electricity: Onshore: [onwind] PV: [solar] + adjust_carrier: # This is the solar+c0.5 thing + enable: false + #solar: + # p_nom_max: + # capital_cost: + # marginal_cost: + + autarky: + enable: false + by_country: false + # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#atlite atlite: default_cutout: europe-2013-era5 @@ -573,6 +620,11 @@ costs: emission_prices: co2: 0. + enable: + emission_prices: false + monthly_prices: false + + # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#clustering clustering: simplify_network: diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index f9fdc3ac..6b9cde24 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -473,10 +473,14 @@ rule prepare_network: links=config["links"], lines=config["lines"], co2base=config["electricity"]["co2base"], + co2limit_enable=config["electricity"].get("co2limit_enable", False), co2limit=config["electricity"]["co2limit"], + gaslimit_enable=config["electricity"].get("gaslimit_enable", False), gaslimit=config["electricity"].get("gaslimit"), max_hours=config["electricity"]["max_hours"], costs=config["costs"], + snapshot_opts=config.get("snapshot_opts",{}), + autarky=config["electricity"].get("autarky",{}), input: RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", tech_costs=COSTS, diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 10a5f821..9845ac13 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -705,6 +705,7 @@ rule build_transport_demand: rule prepare_sector_network: params: + enable_sector=config.get("enable_sector", {}), co2_budget=config["co2_budget"], conventional_carriers=config["existing_capacities"]["conventional_carriers"], foresight=config["foresight"], @@ -717,6 +718,7 @@ rule prepare_sector_network: countries=config["countries"], emissions_scope=config["energy"]["emissions"], eurostat_report_year=config["energy"]["eurostat_report_year"], + snapshot_opts=config.get("snapshot_opts",{}), RDIR=RDIR, input: **build_retro_cost_output, diff --git a/scripts/_helpers.py b/scripts/_helpers.py index fc7bc9e0..714bf33f 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -7,6 +7,7 @@ import contextlib import logging import os import urllib +import re from pathlib import Path import pandas as pd @@ -21,6 +22,19 @@ logger = logging.getLogger(__name__) REGION_COLS = ["geometry", "name", "x", "y", "country"] +def get_opt(opts, expr, flags=None): + """ + Return the first option matching the regular expression. + The regular expression is case-insensitive by default. + """ + if flags is None: + flags = re.IGNORECASE + for o in opts: + match = re.match(expr, o, flags=flags) + if match: + return match.group(0) + return None + # Define a context manager to temporarily mute print statements @contextlib.contextmanager def mute_print(): diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index a5a00a3c..c3fe74b1 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -63,7 +63,7 @@ import re import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging +from _helpers import configure_logging, get_opt from add_electricity import load_costs, update_transmission_costs from pypsa.descriptors import expand_series @@ -71,6 +71,18 @@ idx = pd.IndexSlice logger = logging.getLogger(__name__) +def find_opt(opts, expr): + """ + Return if available the float after the expression. + """ + for o in opts: + if expr in o: + m = re.findall("[0-9]*\.?[0-9]+$", o) + if len(m) > 0: + return True, float(m[0]) + else: + return True, None + return False, None def add_co2limit(n, co2limit, Nyears=1.0): n.add( @@ -296,43 +308,47 @@ if __name__ == "__main__": ) set_line_s_max_pu(n, snakemake.params.lines["s_max_pu"]) + + # temporal averaging + nhours_opts_config = snakemake.params.snapshot_opts.get("average_every_nhours",{}) + nhours_enable_config = nhours_opts_config.get("enable",None) + nhours_config = str(nhours_opts_config.get("hour",None)) + "h" + nhours_wildcard = get_opt(opts, r"^\d+h$") + if nhours_wildcard is not None or (nhours_enable_config and nhours_config is not None): + nhours = nhours_wildcard or nhours_config + n = average_every_nhours(n, nhours) - for o in opts: - m = re.match(r"^\d+h$", o, re.IGNORECASE) - if m is not None: - n = average_every_nhours(n, m.group(0)) - break + # segments with package tsam + time_seg_opts_config = snakemake.params.snapshot_opts.get("time_segmentation",{}) + time_seg_enable_config = nhours_opts_config.get("enable",None) + time_seg_config = str(nhours_opts_config.get("hour",None)) + "seg" + time_seg_wildcard = get_opt(opts, r"^\d+seg$") + if time_seg_wildcard is not None or (time_seg_enable_config and time_seg_config is not None): + time_seg = time_seg_wildcard or time_seg_config + solver_name = snakemake.config["solving"]["solver"]["name"] + n = apply_time_segmentation(n, time_seg, solver_name) - for o in opts: - m = re.match(r"^\d+seg$", o, re.IGNORECASE) - if m is not None: - solver_name = snakemake.config["solving"]["solver"]["name"] - n = apply_time_segmentation(n, m.group(0)[:-3], solver_name) - break + Co2L_config = snakemake.params.co2limit_enable and isinstance(snakemake.params.co2limit,float) + Co2L_wildcard, co2limit_wildcard = find_opt(opts, "Co2L") + if Co2L_wildcard or Co2L_config: + if co2limit_wildcard is not None: # TODO: what if you wat to determine the factor through the wildcard? + co2limit = co2limit_wildcard * snakemake.params.co2base + add_co2limit(n, co2limit, Nyears) + logger.info("Setting CO2 limit according to wildcard value.") + else: + add_co2limit(n, snakemake.params.co2limit, Nyears) + logger.info("Setting CO2 limit according to config value.") - for o in opts: - if "Co2L" in o: - m = re.findall("[0-9]*\.?[0-9]+$", o) - if len(m) > 0: - co2limit = float(m[0]) * snakemake.params.co2base - add_co2limit(n, co2limit, Nyears) - logger.info("Setting CO2 limit according to wildcard value.") - else: - add_co2limit(n, snakemake.params.co2limit, Nyears) - logger.info("Setting CO2 limit according to config value.") - break - - for o in opts: - if "CH4L" in o: - m = re.findall("[0-9]*\.?[0-9]+$", o) - if len(m) > 0: - limit = float(m[0]) * 1e6 - add_gaslimit(n, limit, Nyears) - logger.info("Setting gas usage limit according to wildcard value.") - else: - add_gaslimit(n, snakemake.params.gaslimit, Nyears) - logger.info("Setting gas usage limit according to config value.") - break + CH4L_config = snakemake.params.gaslimit_enable and isinstance(snakemake.params.gaslimit,float) + CH4L_wildcard, gaslimit_wildcard = find_opt(opts, "CH4L") + if CH4L_wildcard or CH4L_config: + if gaslimit_wildcard is not None: # TODO: what if you wat to determine the factor through the wildcard? + gaslimit = gaslimit_wildcard * 1e6 + add_gaslimit(n, gaslimit, Nyears) + logger.info("Setting gas usage limit according to wildcard value.") + else: + add_gaslimit(n, snakemake.params.gaslimit, Nyears) + logger.info("Setting gas usage limit according to config value.") for o in opts: if "+" not in o: @@ -353,21 +369,24 @@ if __name__ == "__main__": sel = c.df.carrier.str.contains(carrier) c.df.loc[sel, attr] *= factor + Ept_config = snakemake.params.costs.get("enable",{}).get("monthly_prices", False) for o in opts: - if "Ept" in o: + if "Ept" in o or Ept_config: logger.info( "Setting time dependent emission prices according spot market price" ) add_dynamic_emission_prices(n) - elif "Ep" in o: - m = re.findall("[0-9]*\.?[0-9]+$", o) - if len(m) > 0: - logger.info("Setting emission prices according to wildcard value.") - add_emission_prices(n, dict(co2=float(m[0]))) - else: - logger.info("Setting emission prices according to config value.") - add_emission_prices(n, snakemake.params.costs["emission_prices"]) - break + Ept_config = True + + Ep_config = snakemake.params.costs.get("enable",{}).get("emission_prices", False) + Ep_wildcard, co2_wildcard = find_opt(opts, "Ep") + if (Ep_wildcard or Ep_config) and not Ept_config: + if co2_wildcard is not None: + logger.info("Setting emission prices according to wildcard value.") + add_emission_prices(n, dict(co2=co2_wildcard)) + else: + logger.info("Setting emission prices according to config value.") + add_emission_prices(n, snakemake.params.costs["emission_prices"]) ll_type, factor = snakemake.wildcards.ll[0], snakemake.wildcards.ll[1:] set_transmission_limit(n, ll_type, factor, costs, Nyears) @@ -380,10 +399,12 @@ if __name__ == "__main__": p_nom_max_ext=snakemake.params.links.get("max_extension", np.inf), ) - if "ATK" in opts: - enforce_autarky(n) - elif "ATKc" in opts: - enforce_autarky(n, only_crossborder=True) + autarky_config = snakemake.params.autarky + if "ATK" in opts or autarky_config.get("enable", False): + only_crossborder = False + if "ATKc" in opts or autarky_config.get("by_country", False): + only_crossborder = True + enforce_autarky(n, only_crossborder=only_crossborder) n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) n.export_to_netcdf(snakemake.output[0]) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 11406bff..af666f5d 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -17,7 +17,7 @@ import numpy as np import pandas as pd import pypsa import xarray as xr -from _helpers import generate_periodic_profiles, update_config_with_sector_opts +from _helpers import generate_periodic_profiles, update_config_with_sector_opts, get_opt from add_electricity import calculate_annuity, sanitize_carriers from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2 from networkx.algorithms import complement @@ -161,11 +161,11 @@ spatial = SimpleNamespace() def emission_sectors_from_opts(opts): sectors = ["electricity"] - if "T" in opts: + if "T" in opts or opts_config.get("land_transport",False): sectors += ["rail non-elec", "road non-elec"] - if "H" in opts: + if "H" in opts or opts_config.get("heating",False): sectors += ["residential non-elec", "services non-elec"] - if "I" in opts: + if "I" in opts or opts_config.get("industry",False): sectors += [ "industrial non-elec", "industrial processes", @@ -174,7 +174,10 @@ def emission_sectors_from_opts(opts): "domestic navigation", "international navigation", ] - if "A" in opts: + + heat_and_industry = opts_config.get("industry",False) and opts_config.get("heating",False) + + if ("I" in opts and "H" in opts and"A" in opts) or (heat_and_industry and opts_config.get("agriculture_machinery",False)): sectors += ["agriculture"] return sectors @@ -3256,27 +3259,39 @@ def set_temporal_aggregation(n, opts, solver_name): """ Aggregate network temporally. """ - for o in opts: - # temporal averaging - m = re.match(r"^\d+h$", o, re.IGNORECASE) - if m is not None: - n = average_every_nhours(n, m.group(0)) - break - # representative snapshots - m = re.match(r"(^\d+)sn$", o, re.IGNORECASE) - if m is not None: - sn = int(m[1]) - logger.info(f"Use every {sn} snapshot as representative") - n.set_snapshots(n.snapshots[::sn]) - n.snapshot_weightings *= sn - break - # segments with package tsam - m = re.match(r"^(\d+)seg$", o, re.IGNORECASE) - if m is not None: - segments = int(m[1]) - logger.info(f"Use temporal segmentation with {segments} segments") - n = apply_time_segmentation(n, segments, solver_name=solver_name) - break + # temporal averaging + nhours_opts_config = snakemake.params.snapshot_opts.get("average_every_nhours",{}) + nhours_enable_config = nhours_opts_config.get("enable",None) + nhours_config = str(nhours_opts_config.get("hour",None)) + "H" + nhours_wildcard = get_opt(opts, r"^\d+h$") + if nhours_wildcard is not None or (nhours_enable_config and nhours_config is not None): + nhours = nhours_wildcard or nhours_config + n = average_every_nhours(n, nhours) + return n + + # representative snapshots + snapshots_opts_config = snakemake.params.snapshot_opts.get("set_snapshots",{}) + snapshots_enable_config = snapshots_opts_config.get("enable",None) + snapshots_config = snapshots_opts_config.get("hour",None) + snapshots_wildcard = get_opt(opts, r"(^\d+)sn$") + if snapshots_wildcard is not None or (snapshots_enable_config and snapshots_config is not None): + sn = int(snapshots_wildcard[:-2]) or snapshots_config + logger.info(f"Use every {sn} snapshot as representative") + n.set_snapshots(n.snapshots[::sn]) + n.snapshot_weightings *= sn + return n + + # segments with package tsam + time_seg_opts_config = snakemake.params.snapshot_opts.get("time_segmentation",{}) + time_seg_enable_config = nhours_opts_config.get("enable",None) + time_seg_config = nhours_opts_config.get("hour",None) + time_seg_wildcard = get_opt(opts, r"^(\d+)seg$") + if time_seg_wildcard is not None or (time_seg_enable_config and time_seg_config is not None): + segments = int(time_seg_wildcard[:-3]) or time_seg_config + logger.info(f"Use temporal segmentation with {segments} segments") + n = apply_time_segmentation(n, segments, solver_name=solver_name) + return n + return n @@ -3303,6 +3318,10 @@ if __name__ == "__main__": opts = snakemake.wildcards.sector_opts.split("-") + opts_config = snakemake.params.enable_sector + + heat_and_industry = opts_config.get("industry",False) and opts_config.get("heating",False) + investment_year = int(snakemake.wildcards.planning_horizons[-4:]) n = pypsa.Network(snakemake.input.network) @@ -3340,51 +3359,53 @@ if __name__ == "__main__": # TODO merge with opts cost adjustment below for o in opts: - if o[:4] == "wave": + if o[:4] == "wave": # TODO: add config wildcard options or depreciated? wave_cost_factor = float(o[4:].replace("p", ".").replace("m", "-")) logger.info( f"Including wave generators with cost factor of {wave_cost_factor}" ) add_wave(n, wave_cost_factor) - if o[:4] == "dist": + if o[:4] == "dist": # TODO: add config wildcard options options["electricity_distribution_grid"] = True options["electricity_distribution_grid_cost_factor"] = float( o[4:].replace("p", ".").replace("m", "-") ) - if o == "biomasstransport": + for o in opts: + if o == "biomasstransport" or opts_config.get("biomass_transport",False): options["biomass_transport"] = True + break - if "nodistrict" in opts: + if "nodistrict" in opts or opts_config.get("no_heat_district",False): options["district_heating"]["progress"] = 0.0 - if "T" in opts: + if "T" in opts or opts_config.get("land_transport",False): add_land_transport(n, costs) - if "H" in opts: + if "H" in opts or opts_config.get("heating",False): add_heat(n, costs) - if "B" in opts: + if "B" in opts or opts_config.get("biomass",False): add_biomass(n, costs) if options["ammonia"]: add_ammonia(n, costs) - if "I" in opts: + if "I" in opts or opts_config.get("industry",False): add_industry(n, costs) - if "I" in opts and "H" in opts: + if ("I" in opts and "H" in opts) or (heat_and_industry and opts_config.get("waste_heat",False)): add_waste_heat(n) - if "A" in opts: # requires H and I + if ("I" in opts and "H" in opts and"A" in opts) or (heat_and_industry and opts_config.get("agriculture_machinery",False)): # requires H and I add_agriculture(n, costs) if options["dac"]: add_dac(n, costs) - if "decentral" in opts: + if "decentral" in opts or opts_config.get("decentral",False): decentral(n) - if "noH2network" in opts: + if "noH2network" in opts or opts_config.get("noH2network",False): remove_h2_network(n) if options["co2network"]: @@ -3399,7 +3420,7 @@ if __name__ == "__main__": limit_type = "config" limit = get(snakemake.params.co2_budget, investment_year) for o in opts: - if "cb" not in o: + if "cb" not in o or opts_config.get("carbon_budget",False) is False: continue limit_type = "carbon budget" fn = "results/" + snakemake.params.RDIR + "csvs/carbon_budget_distribution.csv" @@ -3419,7 +3440,7 @@ if __name__ == "__main__": limit = co2_cap.loc[investment_year] break for o in opts: - if "Co2L" not in o: + if "Co2L" not in o or opts_config.get("co2limit_sector",False) is False: continue limit_type = "wildcard" limit = o[o.find("Co2L") + 4 :] @@ -3428,7 +3449,7 @@ if __name__ == "__main__": logger.info(f"Add CO2 limit from {limit_type}") add_co2limit(n, nyears, limit) - for o in opts: + for o in opts: # TODO: add config wildcard options or depreciated? if not o[:10] == "linemaxext": continue maxext = float(o[10:]) * 1e3