From 823df52309d1a1a44bb449802c2fdbaee8cb0d95 Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Mon, 11 Sep 2023 22:51:31 +0200 Subject: [PATCH 01/15] add backward compatible config in wildcards --- config/config.default.yaml | 52 +++++++++++++ rules/build_electricity.smk | 4 + rules/build_sector.smk | 2 + scripts/_helpers.py | 14 ++++ scripts/prepare_network.py | 119 ++++++++++++++++++------------ scripts/prepare_sector_network.py | 103 ++++++++++++++++---------- 6 files changed, 204 insertions(+), 90 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index b162b75d..fcc74981 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -60,6 +60,14 @@ snapshots: end: "2014-01-01" inclusive: 'left' +snapshot_opts: + average_every_nhours: + enable: false + hour: 2 + time_segmentation: + enable: false + hour: 4380 + # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#enable enable: retrieve: auto @@ -73,6 +81,22 @@ enable: retrieve_natura_raster: true custom_busmap: false +enable_sector: + no_heat_district: false + land_transport: false + heating: false + waste_heat: false + biomass: false + biomass_transport: false + agriculture_machinery: false + industry: false + decentral: false + #wave_energy: false + #wave_energy_factor: + noH2network: false + carbon_budget: false + co2limit_sector: false + # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#co2-budget co2_budget: 2020: 0.701 @@ -83,10 +107,22 @@ co2_budget: 2045: 0.032 2050: 0.000 +co2_budget_opts: + from_descrete_value: + enable: true + from_beta_decay: + enable: false # TODO: move to own rule with sector-opts wildcard? + value: 1 + from_exp_decay: + enable: false + value: 1 + # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#electricity electricity: voltages: [220., 300., 380.] + gaslimit_enable: false gaslimit: false + co2limit_enable: false co2limit: 7.75e+7 co2base: 1.487e+9 agg_p_nom_limits: data/agg_p_nom_minmax.csv @@ -123,6 +159,17 @@ electricity: Onshore: [onwind] PV: [solar] + adjust_carrier: # This is the solar+c0.5 thing + enable: false + #solar: + # p_nom_max: + # capital_cost: + # marginal_cost: + + autarky: + enable: false + by_country: false + # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#atlite atlite: default_cutout: europe-2013-era5 @@ -573,6 +620,11 @@ costs: emission_prices: co2: 0. + enable: + emission_prices: false + monthly_prices: false + + # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#clustering clustering: simplify_network: diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index f9fdc3ac..6b9cde24 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -473,10 +473,14 @@ rule prepare_network: links=config["links"], lines=config["lines"], co2base=config["electricity"]["co2base"], + co2limit_enable=config["electricity"].get("co2limit_enable", False), co2limit=config["electricity"]["co2limit"], + gaslimit_enable=config["electricity"].get("gaslimit_enable", False), gaslimit=config["electricity"].get("gaslimit"), max_hours=config["electricity"]["max_hours"], costs=config["costs"], + snapshot_opts=config.get("snapshot_opts",{}), + autarky=config["electricity"].get("autarky",{}), input: RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", tech_costs=COSTS, diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 10a5f821..9845ac13 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -705,6 +705,7 @@ rule build_transport_demand: rule prepare_sector_network: params: + enable_sector=config.get("enable_sector", {}), co2_budget=config["co2_budget"], conventional_carriers=config["existing_capacities"]["conventional_carriers"], foresight=config["foresight"], @@ -717,6 +718,7 @@ rule prepare_sector_network: countries=config["countries"], emissions_scope=config["energy"]["emissions"], eurostat_report_year=config["energy"]["eurostat_report_year"], + snapshot_opts=config.get("snapshot_opts",{}), RDIR=RDIR, input: **build_retro_cost_output, diff --git a/scripts/_helpers.py b/scripts/_helpers.py index fc7bc9e0..714bf33f 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -7,6 +7,7 @@ import contextlib import logging import os import urllib +import re from pathlib import Path import pandas as pd @@ -21,6 +22,19 @@ logger = logging.getLogger(__name__) REGION_COLS = ["geometry", "name", "x", "y", "country"] +def get_opt(opts, expr, flags=None): + """ + Return the first option matching the regular expression. + The regular expression is case-insensitive by default. + """ + if flags is None: + flags = re.IGNORECASE + for o in opts: + match = re.match(expr, o, flags=flags) + if match: + return match.group(0) + return None + # Define a context manager to temporarily mute print statements @contextlib.contextmanager def mute_print(): diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index a5a00a3c..c3fe74b1 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -63,7 +63,7 @@ import re import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging +from _helpers import configure_logging, get_opt from add_electricity import load_costs, update_transmission_costs from pypsa.descriptors import expand_series @@ -71,6 +71,18 @@ idx = pd.IndexSlice logger = logging.getLogger(__name__) +def find_opt(opts, expr): + """ + Return if available the float after the expression. + """ + for o in opts: + if expr in o: + m = re.findall("[0-9]*\.?[0-9]+$", o) + if len(m) > 0: + return True, float(m[0]) + else: + return True, None + return False, None def add_co2limit(n, co2limit, Nyears=1.0): n.add( @@ -296,43 +308,47 @@ if __name__ == "__main__": ) set_line_s_max_pu(n, snakemake.params.lines["s_max_pu"]) + + # temporal averaging + nhours_opts_config = snakemake.params.snapshot_opts.get("average_every_nhours",{}) + nhours_enable_config = nhours_opts_config.get("enable",None) + nhours_config = str(nhours_opts_config.get("hour",None)) + "h" + nhours_wildcard = get_opt(opts, r"^\d+h$") + if nhours_wildcard is not None or (nhours_enable_config and nhours_config is not None): + nhours = nhours_wildcard or nhours_config + n = average_every_nhours(n, nhours) - for o in opts: - m = re.match(r"^\d+h$", o, re.IGNORECASE) - if m is not None: - n = average_every_nhours(n, m.group(0)) - break + # segments with package tsam + time_seg_opts_config = snakemake.params.snapshot_opts.get("time_segmentation",{}) + time_seg_enable_config = nhours_opts_config.get("enable",None) + time_seg_config = str(nhours_opts_config.get("hour",None)) + "seg" + time_seg_wildcard = get_opt(opts, r"^\d+seg$") + if time_seg_wildcard is not None or (time_seg_enable_config and time_seg_config is not None): + time_seg = time_seg_wildcard or time_seg_config + solver_name = snakemake.config["solving"]["solver"]["name"] + n = apply_time_segmentation(n, time_seg, solver_name) - for o in opts: - m = re.match(r"^\d+seg$", o, re.IGNORECASE) - if m is not None: - solver_name = snakemake.config["solving"]["solver"]["name"] - n = apply_time_segmentation(n, m.group(0)[:-3], solver_name) - break + Co2L_config = snakemake.params.co2limit_enable and isinstance(snakemake.params.co2limit,float) + Co2L_wildcard, co2limit_wildcard = find_opt(opts, "Co2L") + if Co2L_wildcard or Co2L_config: + if co2limit_wildcard is not None: # TODO: what if you wat to determine the factor through the wildcard? + co2limit = co2limit_wildcard * snakemake.params.co2base + add_co2limit(n, co2limit, Nyears) + logger.info("Setting CO2 limit according to wildcard value.") + else: + add_co2limit(n, snakemake.params.co2limit, Nyears) + logger.info("Setting CO2 limit according to config value.") - for o in opts: - if "Co2L" in o: - m = re.findall("[0-9]*\.?[0-9]+$", o) - if len(m) > 0: - co2limit = float(m[0]) * snakemake.params.co2base - add_co2limit(n, co2limit, Nyears) - logger.info("Setting CO2 limit according to wildcard value.") - else: - add_co2limit(n, snakemake.params.co2limit, Nyears) - logger.info("Setting CO2 limit according to config value.") - break - - for o in opts: - if "CH4L" in o: - m = re.findall("[0-9]*\.?[0-9]+$", o) - if len(m) > 0: - limit = float(m[0]) * 1e6 - add_gaslimit(n, limit, Nyears) - logger.info("Setting gas usage limit according to wildcard value.") - else: - add_gaslimit(n, snakemake.params.gaslimit, Nyears) - logger.info("Setting gas usage limit according to config value.") - break + CH4L_config = snakemake.params.gaslimit_enable and isinstance(snakemake.params.gaslimit,float) + CH4L_wildcard, gaslimit_wildcard = find_opt(opts, "CH4L") + if CH4L_wildcard or CH4L_config: + if gaslimit_wildcard is not None: # TODO: what if you wat to determine the factor through the wildcard? + gaslimit = gaslimit_wildcard * 1e6 + add_gaslimit(n, gaslimit, Nyears) + logger.info("Setting gas usage limit according to wildcard value.") + else: + add_gaslimit(n, snakemake.params.gaslimit, Nyears) + logger.info("Setting gas usage limit according to config value.") for o in opts: if "+" not in o: @@ -353,21 +369,24 @@ if __name__ == "__main__": sel = c.df.carrier.str.contains(carrier) c.df.loc[sel, attr] *= factor + Ept_config = snakemake.params.costs.get("enable",{}).get("monthly_prices", False) for o in opts: - if "Ept" in o: + if "Ept" in o or Ept_config: logger.info( "Setting time dependent emission prices according spot market price" ) add_dynamic_emission_prices(n) - elif "Ep" in o: - m = re.findall("[0-9]*\.?[0-9]+$", o) - if len(m) > 0: - logger.info("Setting emission prices according to wildcard value.") - add_emission_prices(n, dict(co2=float(m[0]))) - else: - logger.info("Setting emission prices according to config value.") - add_emission_prices(n, snakemake.params.costs["emission_prices"]) - break + Ept_config = True + + Ep_config = snakemake.params.costs.get("enable",{}).get("emission_prices", False) + Ep_wildcard, co2_wildcard = find_opt(opts, "Ep") + if (Ep_wildcard or Ep_config) and not Ept_config: + if co2_wildcard is not None: + logger.info("Setting emission prices according to wildcard value.") + add_emission_prices(n, dict(co2=co2_wildcard)) + else: + logger.info("Setting emission prices according to config value.") + add_emission_prices(n, snakemake.params.costs["emission_prices"]) ll_type, factor = snakemake.wildcards.ll[0], snakemake.wildcards.ll[1:] set_transmission_limit(n, ll_type, factor, costs, Nyears) @@ -380,10 +399,12 @@ if __name__ == "__main__": p_nom_max_ext=snakemake.params.links.get("max_extension", np.inf), ) - if "ATK" in opts: - enforce_autarky(n) - elif "ATKc" in opts: - enforce_autarky(n, only_crossborder=True) + autarky_config = snakemake.params.autarky + if "ATK" in opts or autarky_config.get("enable", False): + only_crossborder = False + if "ATKc" in opts or autarky_config.get("by_country", False): + only_crossborder = True + enforce_autarky(n, only_crossborder=only_crossborder) n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) n.export_to_netcdf(snakemake.output[0]) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 11406bff..af666f5d 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -17,7 +17,7 @@ import numpy as np import pandas as pd import pypsa import xarray as xr -from _helpers import generate_periodic_profiles, update_config_with_sector_opts +from _helpers import generate_periodic_profiles, update_config_with_sector_opts, get_opt from add_electricity import calculate_annuity, sanitize_carriers from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2 from networkx.algorithms import complement @@ -161,11 +161,11 @@ spatial = SimpleNamespace() def emission_sectors_from_opts(opts): sectors = ["electricity"] - if "T" in opts: + if "T" in opts or opts_config.get("land_transport",False): sectors += ["rail non-elec", "road non-elec"] - if "H" in opts: + if "H" in opts or opts_config.get("heating",False): sectors += ["residential non-elec", "services non-elec"] - if "I" in opts: + if "I" in opts or opts_config.get("industry",False): sectors += [ "industrial non-elec", "industrial processes", @@ -174,7 +174,10 @@ def emission_sectors_from_opts(opts): "domestic navigation", "international navigation", ] - if "A" in opts: + + heat_and_industry = opts_config.get("industry",False) and opts_config.get("heating",False) + + if ("I" in opts and "H" in opts and"A" in opts) or (heat_and_industry and opts_config.get("agriculture_machinery",False)): sectors += ["agriculture"] return sectors @@ -3256,27 +3259,39 @@ def set_temporal_aggregation(n, opts, solver_name): """ Aggregate network temporally. """ - for o in opts: - # temporal averaging - m = re.match(r"^\d+h$", o, re.IGNORECASE) - if m is not None: - n = average_every_nhours(n, m.group(0)) - break - # representative snapshots - m = re.match(r"(^\d+)sn$", o, re.IGNORECASE) - if m is not None: - sn = int(m[1]) - logger.info(f"Use every {sn} snapshot as representative") - n.set_snapshots(n.snapshots[::sn]) - n.snapshot_weightings *= sn - break - # segments with package tsam - m = re.match(r"^(\d+)seg$", o, re.IGNORECASE) - if m is not None: - segments = int(m[1]) - logger.info(f"Use temporal segmentation with {segments} segments") - n = apply_time_segmentation(n, segments, solver_name=solver_name) - break + # temporal averaging + nhours_opts_config = snakemake.params.snapshot_opts.get("average_every_nhours",{}) + nhours_enable_config = nhours_opts_config.get("enable",None) + nhours_config = str(nhours_opts_config.get("hour",None)) + "H" + nhours_wildcard = get_opt(opts, r"^\d+h$") + if nhours_wildcard is not None or (nhours_enable_config and nhours_config is not None): + nhours = nhours_wildcard or nhours_config + n = average_every_nhours(n, nhours) + return n + + # representative snapshots + snapshots_opts_config = snakemake.params.snapshot_opts.get("set_snapshots",{}) + snapshots_enable_config = snapshots_opts_config.get("enable",None) + snapshots_config = snapshots_opts_config.get("hour",None) + snapshots_wildcard = get_opt(opts, r"(^\d+)sn$") + if snapshots_wildcard is not None or (snapshots_enable_config and snapshots_config is not None): + sn = int(snapshots_wildcard[:-2]) or snapshots_config + logger.info(f"Use every {sn} snapshot as representative") + n.set_snapshots(n.snapshots[::sn]) + n.snapshot_weightings *= sn + return n + + # segments with package tsam + time_seg_opts_config = snakemake.params.snapshot_opts.get("time_segmentation",{}) + time_seg_enable_config = nhours_opts_config.get("enable",None) + time_seg_config = nhours_opts_config.get("hour",None) + time_seg_wildcard = get_opt(opts, r"^(\d+)seg$") + if time_seg_wildcard is not None or (time_seg_enable_config and time_seg_config is not None): + segments = int(time_seg_wildcard[:-3]) or time_seg_config + logger.info(f"Use temporal segmentation with {segments} segments") + n = apply_time_segmentation(n, segments, solver_name=solver_name) + return n + return n @@ -3303,6 +3318,10 @@ if __name__ == "__main__": opts = snakemake.wildcards.sector_opts.split("-") + opts_config = snakemake.params.enable_sector + + heat_and_industry = opts_config.get("industry",False) and opts_config.get("heating",False) + investment_year = int(snakemake.wildcards.planning_horizons[-4:]) n = pypsa.Network(snakemake.input.network) @@ -3340,51 +3359,53 @@ if __name__ == "__main__": # TODO merge with opts cost adjustment below for o in opts: - if o[:4] == "wave": + if o[:4] == "wave": # TODO: add config wildcard options or depreciated? wave_cost_factor = float(o[4:].replace("p", ".").replace("m", "-")) logger.info( f"Including wave generators with cost factor of {wave_cost_factor}" ) add_wave(n, wave_cost_factor) - if o[:4] == "dist": + if o[:4] == "dist": # TODO: add config wildcard options options["electricity_distribution_grid"] = True options["electricity_distribution_grid_cost_factor"] = float( o[4:].replace("p", ".").replace("m", "-") ) - if o == "biomasstransport": + for o in opts: + if o == "biomasstransport" or opts_config.get("biomass_transport",False): options["biomass_transport"] = True + break - if "nodistrict" in opts: + if "nodistrict" in opts or opts_config.get("no_heat_district",False): options["district_heating"]["progress"] = 0.0 - if "T" in opts: + if "T" in opts or opts_config.get("land_transport",False): add_land_transport(n, costs) - if "H" in opts: + if "H" in opts or opts_config.get("heating",False): add_heat(n, costs) - if "B" in opts: + if "B" in opts or opts_config.get("biomass",False): add_biomass(n, costs) if options["ammonia"]: add_ammonia(n, costs) - if "I" in opts: + if "I" in opts or opts_config.get("industry",False): add_industry(n, costs) - if "I" in opts and "H" in opts: + if ("I" in opts and "H" in opts) or (heat_and_industry and opts_config.get("waste_heat",False)): add_waste_heat(n) - if "A" in opts: # requires H and I + if ("I" in opts and "H" in opts and"A" in opts) or (heat_and_industry and opts_config.get("agriculture_machinery",False)): # requires H and I add_agriculture(n, costs) if options["dac"]: add_dac(n, costs) - if "decentral" in opts: + if "decentral" in opts or opts_config.get("decentral",False): decentral(n) - if "noH2network" in opts: + if "noH2network" in opts or opts_config.get("noH2network",False): remove_h2_network(n) if options["co2network"]: @@ -3399,7 +3420,7 @@ if __name__ == "__main__": limit_type = "config" limit = get(snakemake.params.co2_budget, investment_year) for o in opts: - if "cb" not in o: + if "cb" not in o or opts_config.get("carbon_budget",False) is False: continue limit_type = "carbon budget" fn = "results/" + snakemake.params.RDIR + "csvs/carbon_budget_distribution.csv" @@ -3419,7 +3440,7 @@ if __name__ == "__main__": limit = co2_cap.loc[investment_year] break for o in opts: - if "Co2L" not in o: + if "Co2L" not in o or opts_config.get("co2limit_sector",False) is False: continue limit_type = "wildcard" limit = o[o.find("Co2L") + 4 :] @@ -3428,7 +3449,7 @@ if __name__ == "__main__": logger.info(f"Add CO2 limit from {limit_type}") add_co2limit(n, nyears, limit) - for o in opts: + for o in opts: # TODO: add config wildcard options or depreciated? if not o[:10] == "linemaxext": continue maxext = float(o[10:]) * 1e3 From 3eed3410448c93426af03e2fce23aaa7db986e73 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 21:03:58 +0000 Subject: [PATCH 02/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_electricity.smk | 4 +- rules/build_sector.smk | 2 +- scripts/_helpers.py | 4 +- scripts/prepare_network.py | 50 +++++++++++------- scripts/prepare_sector_network.py | 84 ++++++++++++++++++------------- 5 files changed, 88 insertions(+), 56 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 6b9cde24..b15796c9 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -479,8 +479,8 @@ rule prepare_network: gaslimit=config["electricity"].get("gaslimit"), max_hours=config["electricity"]["max_hours"], costs=config["costs"], - snapshot_opts=config.get("snapshot_opts",{}), - autarky=config["electricity"].get("autarky",{}), + snapshot_opts=config.get("snapshot_opts", {}), + autarky=config["electricity"].get("autarky", {}), input: RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", tech_costs=COSTS, diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 9845ac13..1bee1ed0 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -718,7 +718,7 @@ rule prepare_sector_network: countries=config["countries"], emissions_scope=config["energy"]["emissions"], eurostat_report_year=config["energy"]["eurostat_report_year"], - snapshot_opts=config.get("snapshot_opts",{}), + snapshot_opts=config.get("snapshot_opts", {}), RDIR=RDIR, input: **build_retro_cost_output, diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 714bf33f..01349e08 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -6,8 +6,8 @@ import contextlib import logging import os -import urllib import re +import urllib from pathlib import Path import pandas as pd @@ -25,6 +25,7 @@ REGION_COLS = ["geometry", "name", "x", "y", "country"] def get_opt(opts, expr, flags=None): """ Return the first option matching the regular expression. + The regular expression is case-insensitive by default. """ if flags is None: @@ -35,6 +36,7 @@ def get_opt(opts, expr, flags=None): return match.group(0) return None + # Define a context manager to temporarily mute print statements @contextlib.contextmanager def mute_print(): diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index c3fe74b1..2a2d73f8 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -71,6 +71,7 @@ idx = pd.IndexSlice logger = logging.getLogger(__name__) + def find_opt(opts, expr): """ Return if available the float after the expression. @@ -84,6 +85,7 @@ def find_opt(opts, expr): return True, None return False, None + def add_co2limit(n, co2limit, Nyears=1.0): n.add( "GlobalConstraint", @@ -308,45 +310,57 @@ if __name__ == "__main__": ) set_line_s_max_pu(n, snakemake.params.lines["s_max_pu"]) - + # temporal averaging - nhours_opts_config = snakemake.params.snapshot_opts.get("average_every_nhours",{}) - nhours_enable_config = nhours_opts_config.get("enable",None) - nhours_config = str(nhours_opts_config.get("hour",None)) + "h" + nhours_opts_config = snakemake.params.snapshot_opts.get("average_every_nhours", {}) + nhours_enable_config = nhours_opts_config.get("enable", None) + nhours_config = str(nhours_opts_config.get("hour", None)) + "h" nhours_wildcard = get_opt(opts, r"^\d+h$") - if nhours_wildcard is not None or (nhours_enable_config and nhours_config is not None): + if nhours_wildcard is not None or ( + nhours_enable_config and nhours_config is not None + ): nhours = nhours_wildcard or nhours_config n = average_every_nhours(n, nhours) # segments with package tsam - time_seg_opts_config = snakemake.params.snapshot_opts.get("time_segmentation",{}) - time_seg_enable_config = nhours_opts_config.get("enable",None) - time_seg_config = str(nhours_opts_config.get("hour",None)) + "seg" + time_seg_opts_config = snakemake.params.snapshot_opts.get("time_segmentation", {}) + time_seg_enable_config = nhours_opts_config.get("enable", None) + time_seg_config = str(nhours_opts_config.get("hour", None)) + "seg" time_seg_wildcard = get_opt(opts, r"^\d+seg$") - if time_seg_wildcard is not None or (time_seg_enable_config and time_seg_config is not None): + if time_seg_wildcard is not None or ( + time_seg_enable_config and time_seg_config is not None + ): time_seg = time_seg_wildcard or time_seg_config solver_name = snakemake.config["solving"]["solver"]["name"] n = apply_time_segmentation(n, time_seg, solver_name) - Co2L_config = snakemake.params.co2limit_enable and isinstance(snakemake.params.co2limit,float) + Co2L_config = snakemake.params.co2limit_enable and isinstance( + snakemake.params.co2limit, float + ) Co2L_wildcard, co2limit_wildcard = find_opt(opts, "Co2L") if Co2L_wildcard or Co2L_config: - if co2limit_wildcard is not None: # TODO: what if you wat to determine the factor through the wildcard? + if ( + co2limit_wildcard is not None + ): # TODO: what if you wat to determine the factor through the wildcard? co2limit = co2limit_wildcard * snakemake.params.co2base add_co2limit(n, co2limit, Nyears) logger.info("Setting CO2 limit according to wildcard value.") - else: + else: add_co2limit(n, snakemake.params.co2limit, Nyears) logger.info("Setting CO2 limit according to config value.") - CH4L_config = snakemake.params.gaslimit_enable and isinstance(snakemake.params.gaslimit,float) + CH4L_config = snakemake.params.gaslimit_enable and isinstance( + snakemake.params.gaslimit, float + ) CH4L_wildcard, gaslimit_wildcard = find_opt(opts, "CH4L") if CH4L_wildcard or CH4L_config: - if gaslimit_wildcard is not None: # TODO: what if you wat to determine the factor through the wildcard? + if ( + gaslimit_wildcard is not None + ): # TODO: what if you wat to determine the factor through the wildcard? gaslimit = gaslimit_wildcard * 1e6 add_gaslimit(n, gaslimit, Nyears) logger.info("Setting gas usage limit according to wildcard value.") - else: + else: add_gaslimit(n, snakemake.params.gaslimit, Nyears) logger.info("Setting gas usage limit according to config value.") @@ -369,7 +383,7 @@ if __name__ == "__main__": sel = c.df.carrier.str.contains(carrier) c.df.loc[sel, attr] *= factor - Ept_config = snakemake.params.costs.get("enable",{}).get("monthly_prices", False) + Ept_config = snakemake.params.costs.get("enable", {}).get("monthly_prices", False) for o in opts: if "Ept" in o or Ept_config: logger.info( @@ -378,13 +392,13 @@ if __name__ == "__main__": add_dynamic_emission_prices(n) Ept_config = True - Ep_config = snakemake.params.costs.get("enable",{}).get("emission_prices", False) + Ep_config = snakemake.params.costs.get("enable", {}).get("emission_prices", False) Ep_wildcard, co2_wildcard = find_opt(opts, "Ep") if (Ep_wildcard or Ep_config) and not Ept_config: if co2_wildcard is not None: logger.info("Setting emission prices according to wildcard value.") add_emission_prices(n, dict(co2=co2_wildcard)) - else: + else: logger.info("Setting emission prices according to config value.") add_emission_prices(n, snakemake.params.costs["emission_prices"]) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index af666f5d..a2ffc3da 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -17,7 +17,7 @@ import numpy as np import pandas as pd import pypsa import xarray as xr -from _helpers import generate_periodic_profiles, update_config_with_sector_opts, get_opt +from _helpers import generate_periodic_profiles, get_opt, update_config_with_sector_opts from add_electricity import calculate_annuity, sanitize_carriers from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2 from networkx.algorithms import complement @@ -161,11 +161,11 @@ spatial = SimpleNamespace() def emission_sectors_from_opts(opts): sectors = ["electricity"] - if "T" in opts or opts_config.get("land_transport",False): + if "T" in opts or opts_config.get("land_transport", False): sectors += ["rail non-elec", "road non-elec"] - if "H" in opts or opts_config.get("heating",False): + if "H" in opts or opts_config.get("heating", False): sectors += ["residential non-elec", "services non-elec"] - if "I" in opts or opts_config.get("industry",False): + if "I" in opts or opts_config.get("industry", False): sectors += [ "industrial non-elec", "industrial processes", @@ -175,9 +175,13 @@ def emission_sectors_from_opts(opts): "international navigation", ] - heat_and_industry = opts_config.get("industry",False) and opts_config.get("heating",False) + heat_and_industry = opts_config.get("industry", False) and opts_config.get( + "heating", False + ) - if ("I" in opts and "H" in opts and"A" in opts) or (heat_and_industry and opts_config.get("agriculture_machinery",False)): + if ("I" in opts and "H" in opts and "A" in opts) or ( + heat_and_industry and opts_config.get("agriculture_machinery", False) + ): sectors += ["agriculture"] return sectors @@ -3260,21 +3264,25 @@ def set_temporal_aggregation(n, opts, solver_name): Aggregate network temporally. """ # temporal averaging - nhours_opts_config = snakemake.params.snapshot_opts.get("average_every_nhours",{}) - nhours_enable_config = nhours_opts_config.get("enable",None) - nhours_config = str(nhours_opts_config.get("hour",None)) + "H" + nhours_opts_config = snakemake.params.snapshot_opts.get("average_every_nhours", {}) + nhours_enable_config = nhours_opts_config.get("enable", None) + nhours_config = str(nhours_opts_config.get("hour", None)) + "H" nhours_wildcard = get_opt(opts, r"^\d+h$") - if nhours_wildcard is not None or (nhours_enable_config and nhours_config is not None): + if nhours_wildcard is not None or ( + nhours_enable_config and nhours_config is not None + ): nhours = nhours_wildcard or nhours_config n = average_every_nhours(n, nhours) return n # representative snapshots - snapshots_opts_config = snakemake.params.snapshot_opts.get("set_snapshots",{}) - snapshots_enable_config = snapshots_opts_config.get("enable",None) - snapshots_config = snapshots_opts_config.get("hour",None) + snapshots_opts_config = snakemake.params.snapshot_opts.get("set_snapshots", {}) + snapshots_enable_config = snapshots_opts_config.get("enable", None) + snapshots_config = snapshots_opts_config.get("hour", None) snapshots_wildcard = get_opt(opts, r"(^\d+)sn$") - if snapshots_wildcard is not None or (snapshots_enable_config and snapshots_config is not None): + if snapshots_wildcard is not None or ( + snapshots_enable_config and snapshots_config is not None + ): sn = int(snapshots_wildcard[:-2]) or snapshots_config logger.info(f"Use every {sn} snapshot as representative") n.set_snapshots(n.snapshots[::sn]) @@ -3282,11 +3290,13 @@ def set_temporal_aggregation(n, opts, solver_name): return n # segments with package tsam - time_seg_opts_config = snakemake.params.snapshot_opts.get("time_segmentation",{}) - time_seg_enable_config = nhours_opts_config.get("enable",None) - time_seg_config = nhours_opts_config.get("hour",None) + time_seg_opts_config = snakemake.params.snapshot_opts.get("time_segmentation", {}) + time_seg_enable_config = nhours_opts_config.get("enable", None) + time_seg_config = nhours_opts_config.get("hour", None) time_seg_wildcard = get_opt(opts, r"^(\d+)seg$") - if time_seg_wildcard is not None or (time_seg_enable_config and time_seg_config is not None): + if time_seg_wildcard is not None or ( + time_seg_enable_config and time_seg_config is not None + ): segments = int(time_seg_wildcard[:-3]) or time_seg_config logger.info(f"Use temporal segmentation with {segments} segments") n = apply_time_segmentation(n, segments, solver_name=solver_name) @@ -3320,7 +3330,9 @@ if __name__ == "__main__": opts_config = snakemake.params.enable_sector - heat_and_industry = opts_config.get("industry",False) and opts_config.get("heating",False) + heat_and_industry = opts_config.get("industry", False) and opts_config.get( + "heating", False + ) investment_year = int(snakemake.wildcards.planning_horizons[-4:]) @@ -3359,53 +3371,57 @@ if __name__ == "__main__": # TODO merge with opts cost adjustment below for o in opts: - if o[:4] == "wave": # TODO: add config wildcard options or depreciated? + if o[:4] == "wave": # TODO: add config wildcard options or depreciated? wave_cost_factor = float(o[4:].replace("p", ".").replace("m", "-")) logger.info( f"Including wave generators with cost factor of {wave_cost_factor}" ) add_wave(n, wave_cost_factor) - if o[:4] == "dist": # TODO: add config wildcard options + if o[:4] == "dist": # TODO: add config wildcard options options["electricity_distribution_grid"] = True options["electricity_distribution_grid_cost_factor"] = float( o[4:].replace("p", ".").replace("m", "-") ) for o in opts: - if o == "biomasstransport" or opts_config.get("biomass_transport",False): + if o == "biomasstransport" or opts_config.get("biomass_transport", False): options["biomass_transport"] = True break - if "nodistrict" in opts or opts_config.get("no_heat_district",False): + if "nodistrict" in opts or opts_config.get("no_heat_district", False): options["district_heating"]["progress"] = 0.0 - if "T" in opts or opts_config.get("land_transport",False): + if "T" in opts or opts_config.get("land_transport", False): add_land_transport(n, costs) - if "H" in opts or opts_config.get("heating",False): + if "H" in opts or opts_config.get("heating", False): add_heat(n, costs) - if "B" in opts or opts_config.get("biomass",False): + if "B" in opts or opts_config.get("biomass", False): add_biomass(n, costs) if options["ammonia"]: add_ammonia(n, costs) - if "I" in opts or opts_config.get("industry",False): + if "I" in opts or opts_config.get("industry", False): add_industry(n, costs) - if ("I" in opts and "H" in opts) or (heat_and_industry and opts_config.get("waste_heat",False)): + if ("I" in opts and "H" in opts) or ( + heat_and_industry and opts_config.get("waste_heat", False) + ): add_waste_heat(n) - if ("I" in opts and "H" in opts and"A" in opts) or (heat_and_industry and opts_config.get("agriculture_machinery",False)): # requires H and I + if ("I" in opts and "H" in opts and "A" in opts) or ( + heat_and_industry and opts_config.get("agriculture_machinery", False) + ): # requires H and I add_agriculture(n, costs) if options["dac"]: add_dac(n, costs) - if "decentral" in opts or opts_config.get("decentral",False): + if "decentral" in opts or opts_config.get("decentral", False): decentral(n) - if "noH2network" in opts or opts_config.get("noH2network",False): + if "noH2network" in opts or opts_config.get("noH2network", False): remove_h2_network(n) if options["co2network"]: @@ -3420,7 +3436,7 @@ if __name__ == "__main__": limit_type = "config" limit = get(snakemake.params.co2_budget, investment_year) for o in opts: - if "cb" not in o or opts_config.get("carbon_budget",False) is False: + if "cb" not in o or opts_config.get("carbon_budget", False) is False: continue limit_type = "carbon budget" fn = "results/" + snakemake.params.RDIR + "csvs/carbon_budget_distribution.csv" @@ -3440,7 +3456,7 @@ if __name__ == "__main__": limit = co2_cap.loc[investment_year] break for o in opts: - if "Co2L" not in o or opts_config.get("co2limit_sector",False) is False: + if "Co2L" not in o or opts_config.get("co2limit_sector", False) is False: continue limit_type = "wildcard" limit = o[o.find("Co2L") + 4 :] @@ -3449,7 +3465,7 @@ if __name__ == "__main__": logger.info(f"Add CO2 limit from {limit_type}") add_co2limit(n, nyears, limit) - for o in opts: # TODO: add config wildcard options or depreciated? + for o in opts: # TODO: add config wildcard options or depreciated? if not o[:10] == "linemaxext": continue maxext = float(o[10:]) * 1e3 From 7be8cc0773380915dccc06c227574528611a2ff9 Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Fri, 15 Sep 2023 11:55:52 +0200 Subject: [PATCH 03/15] undo changes in sector networks and simplify config --- config/config.default.yaml | 50 ++----------- rules/build_electricity.smk | 23 +++++- rules/build_sector.smk | 26 +++++-- rules/postprocess.smk | 6 +- rules/validate.smk | 18 ++++- scripts/prepare_network.py | 26 +++---- scripts/prepare_sector_network.py | 119 ++++++++++-------------------- 7 files changed, 114 insertions(+), 154 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index fcc74981..75de9437 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -59,14 +59,9 @@ snapshots: start: "2013-01-01" end: "2014-01-01" inclusive: 'left' - -snapshot_opts: - average_every_nhours: - enable: false - hour: 2 - time_segmentation: - enable: false - hour: 4380 + resolution: false + segmentation: false + #representative: false # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#enable enable: @@ -81,21 +76,6 @@ enable: retrieve_natura_raster: true custom_busmap: false -enable_sector: - no_heat_district: false - land_transport: false - heating: false - waste_heat: false - biomass: false - biomass_transport: false - agriculture_machinery: false - industry: false - decentral: false - #wave_energy: false - #wave_energy_factor: - noH2network: false - carbon_budget: false - co2limit_sector: false # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#co2-budget co2_budget: @@ -107,16 +87,6 @@ co2_budget: 2045: 0.032 2050: 0.000 -co2_budget_opts: - from_descrete_value: - enable: true - from_beta_decay: - enable: false # TODO: move to own rule with sector-opts wildcard? - value: 1 - from_exp_decay: - enable: false - value: 1 - # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#electricity electricity: voltages: [220., 300., 380.] @@ -159,13 +129,6 @@ electricity: Onshore: [onwind] PV: [solar] - adjust_carrier: # This is the solar+c0.5 thing - enable: false - #solar: - # p_nom_max: - # capital_cost: - # marginal_cost: - autarky: enable: false by_country: false @@ -618,12 +581,9 @@ costs: battery: 0. battery inverter: 0. emission_prices: + enable: false co2: 0. - - enable: - emission_prices: false - monthly_prices: false - + co2_monthly_prices: false # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#clustering clustering: diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index b15796c9..bfbc1bd2 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -20,7 +20,11 @@ if config["enable"].get("prepare_links_p_nom", False): rule build_electricity_demand: params: - snapshots=config["snapshots"], + snapshots={ + "start":config["snapshots"]["start"], + "end":config["snapshots"]["end"], + "inclusive":config["snapshots"]["inclusive"], + }, countries=config["countries"], load=config["load"], input: @@ -61,7 +65,11 @@ rule build_powerplants: rule base_network: params: countries=config["countries"], - snapshots=config["snapshots"], + snapshots={ + "start":config["snapshots"]["start"], + "end":config["snapshots"]["end"], + "inclusive":config["snapshots"]["inclusive"], + }, lines=config["lines"], links=config["links"], transformers=config["transformers"], @@ -144,7 +152,11 @@ if config["enable"].get("build_cutout", False): rule build_cutout: params: - snapshots=config["snapshots"], + snapshots={ + "start":config["snapshots"]["start"], + "end":config["snapshots"]["end"], + "inclusive":config["snapshots"]["inclusive"], + }, cutouts=config["atlite"]["cutouts"], input: regions_onshore=RESOURCES + "regions_onshore.geojson", @@ -470,6 +482,10 @@ rule add_extra_components: rule prepare_network: params: + snapshots={ + "resolution":config["snapshots"].get("resolution", False), + "segmentation":config["snapshots"].get("segmentation", False), + }, links=config["links"], lines=config["lines"], co2base=config["electricity"]["co2base"], @@ -479,7 +495,6 @@ rule prepare_network: gaslimit=config["electricity"].get("gaslimit"), max_hours=config["electricity"]["max_hours"], costs=config["costs"], - snapshot_opts=config.get("snapshot_opts", {}), autarky=config["electricity"].get("autarky", {}), input: RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 1bee1ed0..56a6b532 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -141,7 +141,11 @@ if not (config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]): rule build_heat_demands: params: - snapshots=config["snapshots"], + snapshots={ + "start":config["snapshots"]["start"], + "end":config["snapshots"]["end"], + "inclusive":config["snapshots"]["inclusive"], + }, input: pop_layout=RESOURCES + "pop_layout_{scope}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -163,7 +167,11 @@ rule build_heat_demands: rule build_temperature_profiles: params: - snapshots=config["snapshots"], + snapshots={ + "start":config["snapshots"]["start"], + "end":config["snapshots"]["end"], + "inclusive":config["snapshots"]["inclusive"], + }, input: pop_layout=RESOURCES + "pop_layout_{scope}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -215,7 +223,11 @@ rule build_cop_profiles: rule build_solar_thermal_profiles: params: - snapshots=config["snapshots"], + snapshots={ + "start":config["snapshots"]["start"], + "end":config["snapshots"]["end"], + "inclusive":config["snapshots"]["inclusive"], + }, solar_thermal=config["solar_thermal"], input: pop_layout=RESOURCES + "pop_layout_{scope}.nc", @@ -677,7 +689,11 @@ rule build_shipping_demand: rule build_transport_demand: params: - snapshots=config["snapshots"], + snapshots={ + "start":config["snapshots"]["start"], + "end":config["snapshots"]["end"], + "inclusive":config["snapshots"]["inclusive"], + }, sector=config["sector"], input: clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", @@ -705,7 +721,6 @@ rule build_transport_demand: rule prepare_sector_network: params: - enable_sector=config.get("enable_sector", {}), co2_budget=config["co2_budget"], conventional_carriers=config["existing_capacities"]["conventional_carriers"], foresight=config["foresight"], @@ -718,7 +733,6 @@ rule prepare_sector_network: countries=config["countries"], emissions_scope=config["energy"]["emissions"], eurostat_report_year=config["energy"]["eurostat_report_year"], - snapshot_opts=config.get("snapshot_opts", {}), RDIR=RDIR, input: **build_retro_cost_output, diff --git a/rules/postprocess.smk b/rules/postprocess.smk index cf0038a3..42f0722e 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -55,7 +55,11 @@ rule make_summary: params: foresight=config["foresight"], costs=config["costs"], - snapshots=config["snapshots"], + snapshots={ + "start":config["snapshots"]["start"], + "end":config["snapshots"]["end"], + "inclusive":config["snapshots"]["inclusive"], + }, scenario=config["scenario"], RDIR=RDIR, input: diff --git a/rules/validate.smk b/rules/validate.smk index cfb8c959..ece9bebd 100644 --- a/rules/validate.smk +++ b/rules/validate.smk @@ -17,7 +17,11 @@ rule build_electricity_production: The data is used for validation of the optimization results. """ params: - snapshots=config["snapshots"], + snapshots={ + "start":config["snapshots"]["start"], + "end":config["snapshots"]["end"], + "inclusive":config["snapshots"]["inclusive"], + }, countries=config["countries"], output: RESOURCES + "historical_electricity_production.csv", @@ -35,7 +39,11 @@ rule build_cross_border_flows: The data is used for validation of the optimization results. """ params: - snapshots=config["snapshots"], + snapshots={ + "start":config["snapshots"]["start"], + "end":config["snapshots"]["end"], + "inclusive":config["snapshots"]["inclusive"], + }, countries=config["countries"], input: network=RESOURCES + "networks/base.nc", @@ -55,7 +63,11 @@ rule build_electricity_prices: The data is used for validation of the optimization results. """ params: - snapshots=config["snapshots"], + snapshots={ + "start":config["snapshots"]["start"], + "end":config["snapshots"]["end"], + "inclusive":config["snapshots"]["inclusive"], + }, countries=config["countries"], output: RESOURCES + "historical_electricity_prices.csv", diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 2a2d73f8..f89db024 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -312,24 +312,16 @@ if __name__ == "__main__": set_line_s_max_pu(n, snakemake.params.lines["s_max_pu"]) # temporal averaging - nhours_opts_config = snakemake.params.snapshot_opts.get("average_every_nhours", {}) - nhours_enable_config = nhours_opts_config.get("enable", None) - nhours_config = str(nhours_opts_config.get("hour", None)) + "h" + nhours_config = snakemake.params.snapshots.get("resolution",False) nhours_wildcard = get_opt(opts, r"^\d+h$") - if nhours_wildcard is not None or ( - nhours_enable_config and nhours_config is not None - ): + if nhours_wildcard is not None or isinstance(nhours_config, str): nhours = nhours_wildcard or nhours_config n = average_every_nhours(n, nhours) # segments with package tsam - time_seg_opts_config = snakemake.params.snapshot_opts.get("time_segmentation", {}) - time_seg_enable_config = nhours_opts_config.get("enable", None) - time_seg_config = str(nhours_opts_config.get("hour", None)) + "seg" + time_seg_config = snakemake.params.snapshots.get("segmentation",False) time_seg_wildcard = get_opt(opts, r"^\d+seg$") - if time_seg_wildcard is not None or ( - time_seg_enable_config and time_seg_config is not None - ): + if time_seg_wildcard is not None or isinstance(time_seg_config, str): time_seg = time_seg_wildcard or time_seg_config solver_name = snakemake.config["solving"]["solver"]["name"] n = apply_time_segmentation(n, time_seg, solver_name) @@ -383,7 +375,7 @@ if __name__ == "__main__": sel = c.df.carrier.str.contains(carrier) c.df.loc[sel, attr] *= factor - Ept_config = snakemake.params.costs.get("enable", {}).get("monthly_prices", False) + Ept_config = snakemake.params.costs["emission_prices"].get("co2_monthly_prices", False) for o in opts: if "Ept" in o or Ept_config: logger.info( @@ -392,15 +384,15 @@ if __name__ == "__main__": add_dynamic_emission_prices(n) Ept_config = True - Ep_config = snakemake.params.costs.get("enable", {}).get("emission_prices", False) + Ep_config = snakemake.params.costs["emission_prices"].get("enable", False) Ep_wildcard, co2_wildcard = find_opt(opts, "Ep") if (Ep_wildcard or Ep_config) and not Ept_config: if co2_wildcard is not None: - logger.info("Setting emission prices according to wildcard value.") + logger.info("Setting CO2 prices according to wildcard value.") add_emission_prices(n, dict(co2=co2_wildcard)) else: - logger.info("Setting emission prices according to config value.") - add_emission_prices(n, snakemake.params.costs["emission_prices"]) + logger.info("Setting CO2 prices according to config value.") + add_emission_prices(n, dict(co2=snakemake.params.costs["emission_prices"]["co2"])) ll_type, factor = snakemake.wildcards.ll[0], snakemake.wildcards.ll[1:] set_transmission_limit(n, ll_type, factor, costs, Nyears) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index a2ffc3da..11406bff 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -17,7 +17,7 @@ import numpy as np import pandas as pd import pypsa import xarray as xr -from _helpers import generate_periodic_profiles, get_opt, update_config_with_sector_opts +from _helpers import generate_periodic_profiles, update_config_with_sector_opts from add_electricity import calculate_annuity, sanitize_carriers from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2 from networkx.algorithms import complement @@ -161,11 +161,11 @@ spatial = SimpleNamespace() def emission_sectors_from_opts(opts): sectors = ["electricity"] - if "T" in opts or opts_config.get("land_transport", False): + if "T" in opts: sectors += ["rail non-elec", "road non-elec"] - if "H" in opts or opts_config.get("heating", False): + if "H" in opts: sectors += ["residential non-elec", "services non-elec"] - if "I" in opts or opts_config.get("industry", False): + if "I" in opts: sectors += [ "industrial non-elec", "industrial processes", @@ -174,14 +174,7 @@ def emission_sectors_from_opts(opts): "domestic navigation", "international navigation", ] - - heat_and_industry = opts_config.get("industry", False) and opts_config.get( - "heating", False - ) - - if ("I" in opts and "H" in opts and "A" in opts) or ( - heat_and_industry and opts_config.get("agriculture_machinery", False) - ): + if "A" in opts: sectors += ["agriculture"] return sectors @@ -3263,45 +3256,27 @@ def set_temporal_aggregation(n, opts, solver_name): """ Aggregate network temporally. """ - # temporal averaging - nhours_opts_config = snakemake.params.snapshot_opts.get("average_every_nhours", {}) - nhours_enable_config = nhours_opts_config.get("enable", None) - nhours_config = str(nhours_opts_config.get("hour", None)) + "H" - nhours_wildcard = get_opt(opts, r"^\d+h$") - if nhours_wildcard is not None or ( - nhours_enable_config and nhours_config is not None - ): - nhours = nhours_wildcard or nhours_config - n = average_every_nhours(n, nhours) - return n - - # representative snapshots - snapshots_opts_config = snakemake.params.snapshot_opts.get("set_snapshots", {}) - snapshots_enable_config = snapshots_opts_config.get("enable", None) - snapshots_config = snapshots_opts_config.get("hour", None) - snapshots_wildcard = get_opt(opts, r"(^\d+)sn$") - if snapshots_wildcard is not None or ( - snapshots_enable_config and snapshots_config is not None - ): - sn = int(snapshots_wildcard[:-2]) or snapshots_config - logger.info(f"Use every {sn} snapshot as representative") - n.set_snapshots(n.snapshots[::sn]) - n.snapshot_weightings *= sn - return n - - # segments with package tsam - time_seg_opts_config = snakemake.params.snapshot_opts.get("time_segmentation", {}) - time_seg_enable_config = nhours_opts_config.get("enable", None) - time_seg_config = nhours_opts_config.get("hour", None) - time_seg_wildcard = get_opt(opts, r"^(\d+)seg$") - if time_seg_wildcard is not None or ( - time_seg_enable_config and time_seg_config is not None - ): - segments = int(time_seg_wildcard[:-3]) or time_seg_config - logger.info(f"Use temporal segmentation with {segments} segments") - n = apply_time_segmentation(n, segments, solver_name=solver_name) - return n - + for o in opts: + # temporal averaging + m = re.match(r"^\d+h$", o, re.IGNORECASE) + if m is not None: + n = average_every_nhours(n, m.group(0)) + break + # representative snapshots + m = re.match(r"(^\d+)sn$", o, re.IGNORECASE) + if m is not None: + sn = int(m[1]) + logger.info(f"Use every {sn} snapshot as representative") + n.set_snapshots(n.snapshots[::sn]) + n.snapshot_weightings *= sn + break + # segments with package tsam + m = re.match(r"^(\d+)seg$", o, re.IGNORECASE) + if m is not None: + segments = int(m[1]) + logger.info(f"Use temporal segmentation with {segments} segments") + n = apply_time_segmentation(n, segments, solver_name=solver_name) + break return n @@ -3328,12 +3303,6 @@ if __name__ == "__main__": opts = snakemake.wildcards.sector_opts.split("-") - opts_config = snakemake.params.enable_sector - - heat_and_industry = opts_config.get("industry", False) and opts_config.get( - "heating", False - ) - investment_year = int(snakemake.wildcards.planning_horizons[-4:]) n = pypsa.Network(snakemake.input.network) @@ -3371,57 +3340,51 @@ if __name__ == "__main__": # TODO merge with opts cost adjustment below for o in opts: - if o[:4] == "wave": # TODO: add config wildcard options or depreciated? + if o[:4] == "wave": wave_cost_factor = float(o[4:].replace("p", ".").replace("m", "-")) logger.info( f"Including wave generators with cost factor of {wave_cost_factor}" ) add_wave(n, wave_cost_factor) - if o[:4] == "dist": # TODO: add config wildcard options + if o[:4] == "dist": options["electricity_distribution_grid"] = True options["electricity_distribution_grid_cost_factor"] = float( o[4:].replace("p", ".").replace("m", "-") ) - for o in opts: - if o == "biomasstransport" or opts_config.get("biomass_transport", False): + if o == "biomasstransport": options["biomass_transport"] = True - break - if "nodistrict" in opts or opts_config.get("no_heat_district", False): + if "nodistrict" in opts: options["district_heating"]["progress"] = 0.0 - if "T" in opts or opts_config.get("land_transport", False): + if "T" in opts: add_land_transport(n, costs) - if "H" in opts or opts_config.get("heating", False): + if "H" in opts: add_heat(n, costs) - if "B" in opts or opts_config.get("biomass", False): + if "B" in opts: add_biomass(n, costs) if options["ammonia"]: add_ammonia(n, costs) - if "I" in opts or opts_config.get("industry", False): + if "I" in opts: add_industry(n, costs) - if ("I" in opts and "H" in opts) or ( - heat_and_industry and opts_config.get("waste_heat", False) - ): + if "I" in opts and "H" in opts: add_waste_heat(n) - if ("I" in opts and "H" in opts and "A" in opts) or ( - heat_and_industry and opts_config.get("agriculture_machinery", False) - ): # requires H and I + if "A" in opts: # requires H and I add_agriculture(n, costs) if options["dac"]: add_dac(n, costs) - if "decentral" in opts or opts_config.get("decentral", False): + if "decentral" in opts: decentral(n) - if "noH2network" in opts or opts_config.get("noH2network", False): + if "noH2network" in opts: remove_h2_network(n) if options["co2network"]: @@ -3436,7 +3399,7 @@ if __name__ == "__main__": limit_type = "config" limit = get(snakemake.params.co2_budget, investment_year) for o in opts: - if "cb" not in o or opts_config.get("carbon_budget", False) is False: + if "cb" not in o: continue limit_type = "carbon budget" fn = "results/" + snakemake.params.RDIR + "csvs/carbon_budget_distribution.csv" @@ -3456,7 +3419,7 @@ if __name__ == "__main__": limit = co2_cap.loc[investment_year] break for o in opts: - if "Co2L" not in o or opts_config.get("co2limit_sector", False) is False: + if "Co2L" not in o: continue limit_type = "wildcard" limit = o[o.find("Co2L") + 4 :] @@ -3465,7 +3428,7 @@ if __name__ == "__main__": logger.info(f"Add CO2 limit from {limit_type}") add_co2limit(n, nyears, limit) - for o in opts: # TODO: add config wildcard options or depreciated? + for o in opts: if not o[:10] == "linemaxext": continue maxext = float(o[10:]) * 1e3 From 668ec9efad33af2897484f9bba8dc0c1352603aa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Sep 2023 09:56:15 +0000 Subject: [PATCH 04/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_electricity.smk | 22 +++++++++++----------- rules/build_sector.smk | 24 ++++++++++++------------ rules/postprocess.smk | 6 +++--- rules/validate.smk | 18 +++++++++--------- scripts/prepare_network.py | 12 ++++++++---- 5 files changed, 43 insertions(+), 39 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index bfbc1bd2..c52404bf 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -21,9 +21,9 @@ if config["enable"].get("prepare_links_p_nom", False): rule build_electricity_demand: params: snapshots={ - "start":config["snapshots"]["start"], - "end":config["snapshots"]["end"], - "inclusive":config["snapshots"]["inclusive"], + "start": config["snapshots"]["start"], + "end": config["snapshots"]["end"], + "inclusive": config["snapshots"]["inclusive"], }, countries=config["countries"], load=config["load"], @@ -66,9 +66,9 @@ rule base_network: params: countries=config["countries"], snapshots={ - "start":config["snapshots"]["start"], - "end":config["snapshots"]["end"], - "inclusive":config["snapshots"]["inclusive"], + "start": config["snapshots"]["start"], + "end": config["snapshots"]["end"], + "inclusive": config["snapshots"]["inclusive"], }, lines=config["lines"], links=config["links"], @@ -153,9 +153,9 @@ if config["enable"].get("build_cutout", False): rule build_cutout: params: snapshots={ - "start":config["snapshots"]["start"], - "end":config["snapshots"]["end"], - "inclusive":config["snapshots"]["inclusive"], + "start": config["snapshots"]["start"], + "end": config["snapshots"]["end"], + "inclusive": config["snapshots"]["inclusive"], }, cutouts=config["atlite"]["cutouts"], input: @@ -483,8 +483,8 @@ rule add_extra_components: rule prepare_network: params: snapshots={ - "resolution":config["snapshots"].get("resolution", False), - "segmentation":config["snapshots"].get("segmentation", False), + "resolution": config["snapshots"].get("resolution", False), + "segmentation": config["snapshots"].get("segmentation", False), }, links=config["links"], lines=config["lines"], diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 56a6b532..53a19852 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -142,9 +142,9 @@ if not (config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]): rule build_heat_demands: params: snapshots={ - "start":config["snapshots"]["start"], - "end":config["snapshots"]["end"], - "inclusive":config["snapshots"]["inclusive"], + "start": config["snapshots"]["start"], + "end": config["snapshots"]["end"], + "inclusive": config["snapshots"]["inclusive"], }, input: pop_layout=RESOURCES + "pop_layout_{scope}.nc", @@ -168,9 +168,9 @@ rule build_heat_demands: rule build_temperature_profiles: params: snapshots={ - "start":config["snapshots"]["start"], - "end":config["snapshots"]["end"], - "inclusive":config["snapshots"]["inclusive"], + "start": config["snapshots"]["start"], + "end": config["snapshots"]["end"], + "inclusive": config["snapshots"]["inclusive"], }, input: pop_layout=RESOURCES + "pop_layout_{scope}.nc", @@ -224,9 +224,9 @@ rule build_cop_profiles: rule build_solar_thermal_profiles: params: snapshots={ - "start":config["snapshots"]["start"], - "end":config["snapshots"]["end"], - "inclusive":config["snapshots"]["inclusive"], + "start": config["snapshots"]["start"], + "end": config["snapshots"]["end"], + "inclusive": config["snapshots"]["inclusive"], }, solar_thermal=config["solar_thermal"], input: @@ -690,9 +690,9 @@ rule build_shipping_demand: rule build_transport_demand: params: snapshots={ - "start":config["snapshots"]["start"], - "end":config["snapshots"]["end"], - "inclusive":config["snapshots"]["inclusive"], + "start": config["snapshots"]["start"], + "end": config["snapshots"]["end"], + "inclusive": config["snapshots"]["inclusive"], }, sector=config["sector"], input: diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 42f0722e..02ac219a 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -56,9 +56,9 @@ rule make_summary: foresight=config["foresight"], costs=config["costs"], snapshots={ - "start":config["snapshots"]["start"], - "end":config["snapshots"]["end"], - "inclusive":config["snapshots"]["inclusive"], + "start": config["snapshots"]["start"], + "end": config["snapshots"]["end"], + "inclusive": config["snapshots"]["inclusive"], }, scenario=config["scenario"], RDIR=RDIR, diff --git a/rules/validate.smk b/rules/validate.smk index ece9bebd..1c0fe10a 100644 --- a/rules/validate.smk +++ b/rules/validate.smk @@ -18,9 +18,9 @@ rule build_electricity_production: """ params: snapshots={ - "start":config["snapshots"]["start"], - "end":config["snapshots"]["end"], - "inclusive":config["snapshots"]["inclusive"], + "start": config["snapshots"]["start"], + "end": config["snapshots"]["end"], + "inclusive": config["snapshots"]["inclusive"], }, countries=config["countries"], output: @@ -40,9 +40,9 @@ rule build_cross_border_flows: """ params: snapshots={ - "start":config["snapshots"]["start"], - "end":config["snapshots"]["end"], - "inclusive":config["snapshots"]["inclusive"], + "start": config["snapshots"]["start"], + "end": config["snapshots"]["end"], + "inclusive": config["snapshots"]["inclusive"], }, countries=config["countries"], input: @@ -64,9 +64,9 @@ rule build_electricity_prices: """ params: snapshots={ - "start":config["snapshots"]["start"], - "end":config["snapshots"]["end"], - "inclusive":config["snapshots"]["inclusive"], + "start": config["snapshots"]["start"], + "end": config["snapshots"]["end"], + "inclusive": config["snapshots"]["inclusive"], }, countries=config["countries"], output: diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index f89db024..30ac41bf 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -312,14 +312,14 @@ if __name__ == "__main__": set_line_s_max_pu(n, snakemake.params.lines["s_max_pu"]) # temporal averaging - nhours_config = snakemake.params.snapshots.get("resolution",False) + nhours_config = snakemake.params.snapshots.get("resolution", False) nhours_wildcard = get_opt(opts, r"^\d+h$") if nhours_wildcard is not None or isinstance(nhours_config, str): nhours = nhours_wildcard or nhours_config n = average_every_nhours(n, nhours) # segments with package tsam - time_seg_config = snakemake.params.snapshots.get("segmentation",False) + time_seg_config = snakemake.params.snapshots.get("segmentation", False) time_seg_wildcard = get_opt(opts, r"^\d+seg$") if time_seg_wildcard is not None or isinstance(time_seg_config, str): time_seg = time_seg_wildcard or time_seg_config @@ -375,7 +375,9 @@ if __name__ == "__main__": sel = c.df.carrier.str.contains(carrier) c.df.loc[sel, attr] *= factor - Ept_config = snakemake.params.costs["emission_prices"].get("co2_monthly_prices", False) + Ept_config = snakemake.params.costs["emission_prices"].get( + "co2_monthly_prices", False + ) for o in opts: if "Ept" in o or Ept_config: logger.info( @@ -392,7 +394,9 @@ if __name__ == "__main__": add_emission_prices(n, dict(co2=co2_wildcard)) else: logger.info("Setting CO2 prices according to config value.") - add_emission_prices(n, dict(co2=snakemake.params.costs["emission_prices"]["co2"])) + add_emission_prices( + n, dict(co2=snakemake.params.costs["emission_prices"]["co2"]) + ) ll_type, factor = snakemake.wildcards.ll[0], snakemake.wildcards.ll[1:] set_transmission_limit(n, ll_type, factor, costs, Nyears) From a14e751ed86afc669456cde779c2f82a88cf1230 Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Fri, 15 Sep 2023 12:16:53 +0200 Subject: [PATCH 05/15] fix snapshot bugs --- scripts/base_network.py | 7 ++++++- scripts/build_line_rating.py | 8 +++++++- scripts/build_renewable_profiles.py | 7 ++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/scripts/base_network.py b/scripts/base_network.py index b4ac1d8c..f40b0395 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -709,11 +709,16 @@ def base_network( transformers = _set_electrical_parameters_transformers(transformers, config) links = _set_electrical_parameters_links(links, config, links_p_nom) converters = _set_electrical_parameters_converters(converters, config) + snapshots = { + "start":config["snapshots"]["start"], + "end":config["snapshots"]["end"], + "inclusive":config["snapshots"]["inclusive"], + } n = pypsa.Network() n.name = "PyPSA-Eur" - n.set_snapshots(pd.date_range(freq="h", **config["snapshots"])) + n.set_snapshots(pd.date_range(freq="h", **snapshots)) n.madd("Carrier", ["AC", "DC"]) n.import_components_from_dataframe(buses, "Bus") diff --git a/scripts/build_line_rating.py b/scripts/build_line_rating.py index 7f842d43..cf6b56c4 100755 --- a/scripts/build_line_rating.py +++ b/scripts/build_line_rating.py @@ -148,8 +148,14 @@ if __name__ == "__main__": ) configure_logging(snakemake) + snapshots = { + "start":snakemake.config["snapshots"]["start"], + "end":snakemake.config["snapshots"]["end"], + "inclusive":snakemake.config["snapshots"]["inclusive"], + } + n = pypsa.Network(snakemake.input.base_network) - time = pd.date_range(freq="h", **snakemake.config["snapshots"]) + time = pd.date_range(freq="h", **snapshots) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) da = calculate_line_rating(n, cutout) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 7b08325b..863ba4e4 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -211,6 +211,11 @@ if __name__ == "__main__": correction_factor = params.get("correction_factor", 1.0) capacity_per_sqkm = params["capacity_per_sqkm"] p_nom_max_meth = params.get("potential", "conservative") + snapshots = { + "start":snakemake.config["snapshots"]["start"], + "end":snakemake.config["snapshots"]["end"], + "inclusive":snakemake.config["snapshots"]["inclusive"], + } if isinstance(params.get("corine", {}), list): params["corine"] = {"grid_codes": params["corine"]} @@ -223,7 +228,7 @@ if __name__ == "__main__": else: client = None - sns = pd.date_range(freq="h", **snakemake.config["snapshots"]) + sns = pd.date_range(freq="h", **snapshots) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=sns) regions = gpd.read_file(snakemake.input.regions) assert not regions.empty, ( From b2c19eda40e6409257e616ccc9995ed3063733bd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Sep 2023 10:18:01 +0000 Subject: [PATCH 06/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/base_network.py | 8 ++++---- scripts/build_line_rating.py | 8 ++++---- scripts/build_renewable_profiles.py | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/base_network.py b/scripts/base_network.py index f40b0395..a176028f 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -710,10 +710,10 @@ def base_network( links = _set_electrical_parameters_links(links, config, links_p_nom) converters = _set_electrical_parameters_converters(converters, config) snapshots = { - "start":config["snapshots"]["start"], - "end":config["snapshots"]["end"], - "inclusive":config["snapshots"]["inclusive"], - } + "start": config["snapshots"]["start"], + "end": config["snapshots"]["end"], + "inclusive": config["snapshots"]["inclusive"], + } n = pypsa.Network() n.name = "PyPSA-Eur" diff --git a/scripts/build_line_rating.py b/scripts/build_line_rating.py index cf6b56c4..51c779e5 100755 --- a/scripts/build_line_rating.py +++ b/scripts/build_line_rating.py @@ -149,10 +149,10 @@ if __name__ == "__main__": configure_logging(snakemake) snapshots = { - "start":snakemake.config["snapshots"]["start"], - "end":snakemake.config["snapshots"]["end"], - "inclusive":snakemake.config["snapshots"]["inclusive"], - } + "start": snakemake.config["snapshots"]["start"], + "end": snakemake.config["snapshots"]["end"], + "inclusive": snakemake.config["snapshots"]["inclusive"], + } n = pypsa.Network(snakemake.input.base_network) time = pd.date_range(freq="h", **snapshots) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 863ba4e4..1d3b9956 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -212,10 +212,10 @@ if __name__ == "__main__": capacity_per_sqkm = params["capacity_per_sqkm"] p_nom_max_meth = params.get("potential", "conservative") snapshots = { - "start":snakemake.config["snapshots"]["start"], - "end":snakemake.config["snapshots"]["end"], - "inclusive":snakemake.config["snapshots"]["inclusive"], - } + "start": snakemake.config["snapshots"]["start"], + "end": snakemake.config["snapshots"]["end"], + "inclusive": snakemake.config["snapshots"]["inclusive"], + } if isinstance(params.get("corine", {}), list): params["corine"] = {"grid_codes": params["corine"]} From 01cd1bbb74a5af95b89f9b492ffd0c6190f3d1dd Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Sat, 23 Sep 2023 18:39:11 +0200 Subject: [PATCH 07/15] add Fabians suggestion --- rules/build_electricity.smk | 21 ++++--------- rules/build_sector.smk | 24 +++------------ rules/postprocess.smk | 6 +--- rules/validate.smk | 18 ++--------- scripts/base_network.py | 6 +--- scripts/build_line_rating.py | 6 +--- scripts/build_renewable_profiles.py | 6 +--- scripts/prepare_network.py | 48 ++++++++++++----------------- 8 files changed, 36 insertions(+), 99 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index c52404bf..dc5ce1c6 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -20,11 +20,7 @@ if config["enable"].get("prepare_links_p_nom", False): rule build_electricity_demand: params: - snapshots={ - "start": config["snapshots"]["start"], - "end": config["snapshots"]["end"], - "inclusive": config["snapshots"]["inclusive"], - }, + snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, countries=config["countries"], load=config["load"], input: @@ -65,11 +61,7 @@ rule build_powerplants: rule base_network: params: countries=config["countries"], - snapshots={ - "start": config["snapshots"]["start"], - "end": config["snapshots"]["end"], - "inclusive": config["snapshots"]["inclusive"], - }, + snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, lines=config["lines"], links=config["links"], transformers=config["transformers"], @@ -152,11 +144,7 @@ if config["enable"].get("build_cutout", False): rule build_cutout: params: - snapshots={ - "start": config["snapshots"]["start"], - "end": config["snapshots"]["end"], - "inclusive": config["snapshots"]["inclusive"], - }, + snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, cutouts=config["atlite"]["cutouts"], input: regions_onshore=RESOURCES + "regions_onshore.geojson", @@ -220,6 +208,7 @@ rule build_ship_raster: rule build_renewable_profiles: params: + snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, renewable=config["renewable"], input: base_network=RESOURCES + "networks/base.nc", @@ -310,6 +299,8 @@ rule build_hydro_profile: if config["lines"]["dynamic_line_rating"]["activate"]: rule build_line_rating: + params: + snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, input: base_network=RESOURCES + "networks/base.nc", cutout="cutouts/" diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 53a19852..c148bc9f 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -141,11 +141,7 @@ if not (config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]): rule build_heat_demands: params: - snapshots={ - "start": config["snapshots"]["start"], - "end": config["snapshots"]["end"], - "inclusive": config["snapshots"]["inclusive"], - }, + snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, input: pop_layout=RESOURCES + "pop_layout_{scope}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -167,11 +163,7 @@ rule build_heat_demands: rule build_temperature_profiles: params: - snapshots={ - "start": config["snapshots"]["start"], - "end": config["snapshots"]["end"], - "inclusive": config["snapshots"]["inclusive"], - }, + snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, input: pop_layout=RESOURCES + "pop_layout_{scope}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -223,11 +215,7 @@ rule build_cop_profiles: rule build_solar_thermal_profiles: params: - snapshots={ - "start": config["snapshots"]["start"], - "end": config["snapshots"]["end"], - "inclusive": config["snapshots"]["inclusive"], - }, + snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, solar_thermal=config["solar_thermal"], input: pop_layout=RESOURCES + "pop_layout_{scope}.nc", @@ -689,11 +677,7 @@ rule build_shipping_demand: rule build_transport_demand: params: - snapshots={ - "start": config["snapshots"]["start"], - "end": config["snapshots"]["end"], - "inclusive": config["snapshots"]["inclusive"], - }, + snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, sector=config["sector"], input: clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 02ac219a..795ea1b1 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -55,11 +55,7 @@ rule make_summary: params: foresight=config["foresight"], costs=config["costs"], - snapshots={ - "start": config["snapshots"]["start"], - "end": config["snapshots"]["end"], - "inclusive": config["snapshots"]["inclusive"], - }, + snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, scenario=config["scenario"], RDIR=RDIR, input: diff --git a/rules/validate.smk b/rules/validate.smk index 1c0fe10a..0fa1f607 100644 --- a/rules/validate.smk +++ b/rules/validate.smk @@ -17,11 +17,7 @@ rule build_electricity_production: The data is used for validation of the optimization results. """ params: - snapshots={ - "start": config["snapshots"]["start"], - "end": config["snapshots"]["end"], - "inclusive": config["snapshots"]["inclusive"], - }, + snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, countries=config["countries"], output: RESOURCES + "historical_electricity_production.csv", @@ -39,11 +35,7 @@ rule build_cross_border_flows: The data is used for validation of the optimization results. """ params: - snapshots={ - "start": config["snapshots"]["start"], - "end": config["snapshots"]["end"], - "inclusive": config["snapshots"]["inclusive"], - }, + snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, countries=config["countries"], input: network=RESOURCES + "networks/base.nc", @@ -63,11 +55,7 @@ rule build_electricity_prices: The data is used for validation of the optimization results. """ params: - snapshots={ - "start": config["snapshots"]["start"], - "end": config["snapshots"]["end"], - "inclusive": config["snapshots"]["inclusive"], - }, + snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]}, countries=config["countries"], output: RESOURCES + "historical_electricity_prices.csv", diff --git a/scripts/base_network.py b/scripts/base_network.py index a176028f..372f9a20 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -709,11 +709,7 @@ def base_network( transformers = _set_electrical_parameters_transformers(transformers, config) links = _set_electrical_parameters_links(links, config, links_p_nom) converters = _set_electrical_parameters_converters(converters, config) - snapshots = { - "start": config["snapshots"]["start"], - "end": config["snapshots"]["end"], - "inclusive": config["snapshots"]["inclusive"], - } + snapshots = snakemake.params.snapshots n = pypsa.Network() n.name = "PyPSA-Eur" diff --git a/scripts/build_line_rating.py b/scripts/build_line_rating.py index 51c779e5..1767ebc8 100755 --- a/scripts/build_line_rating.py +++ b/scripts/build_line_rating.py @@ -148,11 +148,7 @@ if __name__ == "__main__": ) configure_logging(snakemake) - snapshots = { - "start": snakemake.config["snapshots"]["start"], - "end": snakemake.config["snapshots"]["end"], - "inclusive": snakemake.config["snapshots"]["inclusive"], - } + snapshots = snakemake.params.snapshots n = pypsa.Network(snakemake.input.base_network) time = pd.date_range(freq="h", **snapshots) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 1d3b9956..a8432219 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -211,11 +211,7 @@ if __name__ == "__main__": correction_factor = params.get("correction_factor", 1.0) capacity_per_sqkm = params["capacity_per_sqkm"] p_nom_max_meth = params.get("potential", "conservative") - snapshots = { - "start": snakemake.config["snapshots"]["start"], - "end": snakemake.config["snapshots"]["end"], - "inclusive": snakemake.config["snapshots"]["inclusive"], - } + snapshots = snakemake.params.snapshots if isinstance(params.get("corine", {}), list): params["corine"] = {"grid_codes": params["corine"]} diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 30ac41bf..2a15cb0c 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -314,26 +314,22 @@ if __name__ == "__main__": # temporal averaging nhours_config = snakemake.params.snapshots.get("resolution", False) nhours_wildcard = get_opt(opts, r"^\d+h$") - if nhours_wildcard is not None or isinstance(nhours_config, str): - nhours = nhours_wildcard or nhours_config + nhours = nhours_wildcard or nhours_config + if nhours: n = average_every_nhours(n, nhours) # segments with package tsam time_seg_config = snakemake.params.snapshots.get("segmentation", False) time_seg_wildcard = get_opt(opts, r"^\d+seg$") - if time_seg_wildcard is not None or isinstance(time_seg_config, str): - time_seg = time_seg_wildcard or time_seg_config + time_seg = time_seg_wildcard or time_seg_config + if time_seg: solver_name = snakemake.config["solving"]["solver"]["name"] n = apply_time_segmentation(n, time_seg, solver_name) - Co2L_config = snakemake.params.co2limit_enable and isinstance( - snakemake.params.co2limit, float - ) + Co2L_config = snakemake.params.co2limit_enable Co2L_wildcard, co2limit_wildcard = find_opt(opts, "Co2L") if Co2L_wildcard or Co2L_config: - if ( - co2limit_wildcard is not None - ): # TODO: what if you wat to determine the factor through the wildcard? + if co2limit_wildcard is not None: co2limit = co2limit_wildcard * snakemake.params.co2base add_co2limit(n, co2limit, Nyears) logger.info("Setting CO2 limit according to wildcard value.") @@ -341,14 +337,10 @@ if __name__ == "__main__": add_co2limit(n, snakemake.params.co2limit, Nyears) logger.info("Setting CO2 limit according to config value.") - CH4L_config = snakemake.params.gaslimit_enable and isinstance( - snakemake.params.gaslimit, float - ) + CH4L_config = snakemake.params.gaslimit_enable CH4L_wildcard, gaslimit_wildcard = find_opt(opts, "CH4L") if CH4L_wildcard or CH4L_config: - if ( - gaslimit_wildcard is not None - ): # TODO: what if you wat to determine the factor through the wildcard? + if gaslimit_wildcard is not None: gaslimit = gaslimit_wildcard * 1e6 add_gaslimit(n, gaslimit, Nyears) logger.info("Setting gas usage limit according to wildcard value.") @@ -375,20 +367,18 @@ if __name__ == "__main__": sel = c.df.carrier.str.contains(carrier) c.df.loc[sel, attr] *= factor - Ept_config = snakemake.params.costs["emission_prices"].get( - "co2_monthly_prices", False - ) - for o in opts: - if "Ept" in o or Ept_config: - logger.info( - "Setting time dependent emission prices according spot market price" - ) - add_dynamic_emission_prices(n) - Ept_config = True - - Ep_config = snakemake.params.costs["emission_prices"].get("enable", False) + emission_prices = snakemake.params.costs["emission_prices"] + Ept_config = emission_prices.get("co2_monthly_prices", False) + Ept_wildcard = "Ept" in opts + Ep_config = emission_prices.get("enable", False) Ep_wildcard, co2_wildcard = find_opt(opts, "Ep") - if (Ep_wildcard or Ep_config) and not Ept_config: + + if Ept_wildcard or Ept_config: + logger.info( + "Setting time dependent emission prices according spot market price" + ) + add_dynamic_emission_prices(n) + elif Ep_wildcard or Ep_config: if co2_wildcard is not None: logger.info("Setting CO2 prices according to wildcard value.") add_emission_prices(n, dict(co2=co2_wildcard)) From 05535590348118ef14a17652b5f7f7f8635bfa3b Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Sat, 23 Sep 2023 19:00:01 +0200 Subject: [PATCH 08/15] add documentations --- doc/configtables/costs.csv | 20 +++++++++++--------- doc/configtables/electricity.csv | 5 +++++ doc/configtables/snapshots.csv | 10 ++++++---- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/doc/configtables/costs.csv b/doc/configtables/costs.csv index 9797d77e..e307985d 100644 --- a/doc/configtables/costs.csv +++ b/doc/configtables/costs.csv @@ -1,9 +1,11 @@ -,Unit,Values,Description -year,--,"YYYY; e.g. '2030'","Year for which to retrieve cost assumptions of ``resources/costs.csv``." -version,--,"vX.X.X; e.g. 'v0.5.0'","Version of ``technology-data`` repository to use." -rooftop_share,--,float,"Share of rooftop PV when calculating capital cost of solar (joint rooftop and utility-scale PV)." -fill_values,--,float,"Default values if not specified for a technology in ``resources/costs.csv``." -capital_cost,EUR/MW,"Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.","For the given technologies, assumptions about their capital investment costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``." -marginal_cost,EUR/MWh,"Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.","For the given technologies, assumptions about their marginal operating costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``." -emission_prices,,,"Specify exogenous prices for emission types listed in ``network.carriers`` to marginal costs." --- co2,EUR/t,float,"Exogenous price of carbon-dioxide added to the marginal costs of fossil-fuelled generators according to their carbon intensity. Added through the keyword ``Ep`` in the ``{opts}`` wildcard only in the rule :mod:`prepare_network``." +,Unit,Values,Description +year,--,YYYY; e.g. '2030',Year for which to retrieve cost assumptions of ``resources/costs.csv``. +version,--,vX.X.X; e.g. 'v0.5.0',Version of ``technology-data`` repository to use. +rooftop_share,--,float,Share of rooftop PV when calculating capital cost of solar (joint rooftop and utility-scale PV). +fill_values,--,float,Default values if not specified for a technology in ``resources/costs.csv``. +capital_cost,EUR/MW,Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.,"For the given technologies, assumptions about their capital investment costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``." +marginal_cost,EUR/MWh,Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.,"For the given technologies, assumptions about their marginal operating costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``." +emission_prices,,,Specify exogenous prices for emission types listed in ``network.carriers`` to marginal costs. +-- enable,bool,true or false,Add cost for a carbon-dioxide price configured in ``costs: emission_prices: co2`` to ``marginal_cost`` of generators (other emission types listed in ``network.carriers`` possible as well) +-- co2,EUR/t,float,Exogenous price of carbon-dioxide added to the marginal costs of fossil-fuelled generators according to their carbon intensity. Added through the keyword ``Ep`` in the ``{opts}`` wildcard only in the rule :mod:`prepare_network``. +-- co2_monthly_price,bool,true or false,Add monthly cost for a carbon-dioxide price based on historical values built by the rule ``build_monthly_prices`` diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index 4c04fee6..00eec0c4 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -1,6 +1,8 @@ ,Unit,Values,Description voltages,kV,"Any subset of {220., 300., 380.}",Voltage levels to consider +gaslimit_enable,bool,true or false,Add an overall absolute gas limit configured in ``electricity: gaslimit``. gaslimit,MWhth,float or false,Global gas usage limit +co2limit_enable,bool,true or false,Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``. co2limit,:math:`t_{CO_2-eq}/a`,float,Cap on total annual system carbon dioxide emissions co2base,:math:`t_{CO_2-eq}/a`,float,Reference value of total annual system carbon dioxide emissions if relative emission reduction target is specified in ``{opts}`` wildcard. agg_p_nom_limits,file,path,Reference to ``.csv`` file specifying per carrier generator nominal capacity constraints for individual countries if ``'CCL'`` is in ``{opts}`` wildcard. Defaults to ``data/agg_p_nom_minmax.csv``. @@ -34,3 +36,6 @@ estimate_renewable_capacities,,, -- -- Offshore,--,"Any subset of {offwind-ac, offwind-dc}","List of PyPSA-Eur carriers that is considered as (IRENA, OPSD) onshore technology." -- -- Offshore,--,{onwind},"List of PyPSA-Eur carriers that is considered as (IRENA, OPSD) offshore technology." -- -- PV,--,{solar},"List of PyPSA-Eur carriers that is considered as (IRENA, OPSD) PV technology." +autarky,,, +-- enable,bool,true or false,Require each node to be autarkic by removing all lines and links. +-- by_country,bool,true or false,Require each country to be autarkic by removing all cross-border lines and links. ``electricity: autarky`` must be enabled. diff --git a/doc/configtables/snapshots.csv b/doc/configtables/snapshots.csv index d60c78dc..769e6cb0 100644 --- a/doc/configtables/snapshots.csv +++ b/doc/configtables/snapshots.csv @@ -1,4 +1,6 @@ -,Unit,Values,Description -start,--,"str or datetime-like; e.g. YYYY-MM-DD","Left bound of date range" -end,--,"str or datetime-like; e.g. YYYY-MM-DD","Right bound of date range" -inclusive,--,"One of {'neither', 'both', ‘left’, ‘right’}","Make the time interval closed to the ``left``, ``right``, or both sides ``both`` or neither side ``None``." +,Unit,Values,Description +start,--,str or datetime-like; e.g. YYYY-MM-DD,Left bound of date range +end,--,str or datetime-like; e.g. YYYY-MM-DD,Right bound of date range +inclusive,--,"One of {'neither', 'both', ‘left’, ‘right’}","Make the time interval closed to the ``left``, ``right``, or both sides ``both`` or neither side ``None``." +resolution ,--,"{false,``nH``; i.e. ``2H``-``6H``}",Resample the time-resolution by averaging over every ``n`` snapshots +segmentation,--,"{false,``nSEG``; e.g. ``4380SEG``}","Apply time series segmentation with `tsam `_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load." From 0d1339b1d3fabebf22cc1f5e3bfe5fcda6677d70 Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Wed, 27 Sep 2023 08:22:28 +0200 Subject: [PATCH 09/15] add wildcard options in config for solve network --- config/config.default.yaml | 6 ++++++ doc/configtables/opts.csv | 26 +++++++++++++------------- doc/configtables/solving.csv | 5 +++++ scripts/solve_network.py | 21 ++++++++++++++------- 4 files changed, 38 insertions(+), 20 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 75de9437..620c04af 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -622,6 +622,12 @@ solving: transmission_losses: 0 linearized_unit_commitment: true horizon: 365 + + constraints: + CCL: false + EQ: false + BAU: false + SAFE: false solver: name: gurobi diff --git a/doc/configtables/opts.csv b/doc/configtables/opts.csv index 8c8a706f..b133c718 100644 --- a/doc/configtables/opts.csv +++ b/doc/configtables/opts.csv @@ -1,13 +1,13 @@ -Trigger, Description, Definition, Status -``nH``; i.e. ``2H``-``6H``, Resample the time-resolution by averaging over every ``n`` snapshots, ``prepare_network``: `average_every_nhours() `_ and its `caller `__), In active use -``nSEG``; e.g. ``4380SEG``, "Apply time series segmentation with `tsam `_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load.", ``prepare_network``: apply_time_segmentation(), In active use -``Co2L``, Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``. If a float is appended an overall emission limit relative to the emission level given in ``electricity: co2base`` is added (e.g. ``Co2L0.05`` limits emissisions to 5% of what is given in ``electricity: co2base``), ``prepare_network``: `add_co2limit() `_ and its `caller `__, In active use -``Ep``, Add cost for a carbon-dioxide price configured in ``costs: emission_prices: co2`` to ``marginal_cost`` of generators (other emission types listed in ``network.carriers`` possible as well), ``prepare_network``: `add_emission_prices() `_ and its `caller `__, In active use -``Ept``, Add monthly cost for a carbon-dioxide price based on historical values built by the rule ``build_monthly_prices``, In active use -``CCL``, Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``., ``solve_network``, In active use -``EQ``, "Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption.", ``solve_network``, In active use -``ATK``, "Require each node to be autarkic. Example: ``ATK`` removes all lines and links. ``ATKc`` removes all cross-border lines and links.", ``prepare_network``, In active use -``BAU``, Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities``, ``solve_network``: `add_opts_constraints() `__, Untested -``SAFE``, Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network., ``solve_network`` `add_opts_constraints() `__, Untested -``carrier+{c|p|m}factor``,"Alter the capital cost (``c``), installable potential (``p``) or marginal costs (``m``) of a carrier by a factor. Example: ``solar+c0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use -``CH4L``,"Add an overall absolute gas limit. If configured in ``electricity: gaslimit`` it is given in MWh thermal, if a float is appended, the overall gaslimit is assumed to be given in TWh thermal (e.g. ``CH4L200`` limits gas dispatch to 200 TWh termal)", ``prepare_network``: ``add_gaslimit()``, In active use +Trigger, Description, Definition, Status +``nH``; i.e. ``2H``-``6H``, Resample the time-resolution by averaging over every ``n`` snapshots, ``prepare_network``: `average_every_nhours() `_ and its `caller `__), In active use +``nSEG``; e.g. ``4380SEG``,"Apply time series segmentation with `tsam `_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load.", ``prepare_network``: apply_time_segmentation(), In active use +``Co2L``,Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``. If a float is appended an overall emission limit relative to the emission level given in ``electricity: co2base`` is added (e.g. ``Co2L0.05`` limits emissisions to 5% of what is given in ``electricity: co2base``), ``prepare_network``: `add_co2limit() `_ and its `caller `__, In active use +``Ep``,Add cost for a carbon-dioxide price configured in ``costs: emission_prices: co2`` to ``marginal_cost`` of generators (other emission types listed in ``network.carriers`` possible as well), ``prepare_network``: `add_emission_prices() `_ and its `caller `__, In active use +``Ept``,Add monthly cost for a carbon-dioxide price based on historical values built by the rule ``build_monthly_prices``, In active use, +``CCL``,Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``., ``solve_network``, In active use +``EQ``,Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption., ``solve_network``, In active use +``ATK``,Require each node to be autarkic. Example: ``ATK`` removes all lines and links. ``ATKc`` removes all cross-border lines and links., ``prepare_network``, In active use +``BAU``,Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities``, ``solve_network``: `add_opts_constraints() `__, Untested +``SAFE``,Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network., ``solve_network`` `add_opts_constraints() `__, Untested +``carrier+{c|p|m}factor``,"Alter the capital cost (``c``), installable potential (``p``) or marginal costs (``m``) of a carrier by a factor. Example: ``solar+c0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use +``CH4L``,"Add an overall absolute gas limit. If configured in ``electricity: gaslimit`` it is given in MWh thermal, if a float is appended, the overall gaslimit is assumed to be given in TWh thermal (e.g. ``CH4L200`` limits gas dispatch to 200 TWh termal)", ``prepare_network``: ``add_gaslimit()``, In active use diff --git a/doc/configtables/solving.csv b/doc/configtables/solving.csv index 45d50d84..344bf73f 100644 --- a/doc/configtables/solving.csv +++ b/doc/configtables/solving.csv @@ -12,6 +12,11 @@ options,,, -- transmission_losses,int,[0-9],"Add piecewise linear approximation of transmission losses based on n tangents. Defaults to 0, which means losses are ignored." -- linearized_unit_commitment,bool,"{'true','false'}",Whether to optimise using the linearized unit commitment formulation. -- horizon,--,int,Number of snapshots to consider in each iteration. Defaults to 100. +constraints ,,, +-- CCL,bool,"{'true','false'}",Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``. +-- EQ,bool/string,"{'false',`EQn(c| )``; i.e. ``EQ0.5``-``EQ0.7c``}",Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption. +-- BAU,bool,"{'true','false'}",Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities`` +-- SAFE,bool,"{'true','false'}",Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network. solver,,, -- name,--,"One of {'gurobi', 'cplex', 'cbc', 'glpk', 'ipopt'}; potentially more possible",Solver to use for optimisation problems in the workflow; e.g. clustering and linear optimal power flow. -- options,--,Key listed under ``solver_options``.,Link to specific parameter settings. diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 836544b4..c1c170ce 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -33,7 +33,7 @@ import numpy as np import pandas as pd import pypsa import xarray as xr -from _helpers import configure_logging, update_config_with_sector_opts +from _helpers import configure_logging, update_config_with_sector_opts, get_opt logger = logging.getLogger(__name__) pypsa.pf.logger.setLevel(logging.WARNING) @@ -542,6 +542,7 @@ def add_chp_constraints(n): # back-pressure if not electric.empty: lhs = ( + p.loc[:, heat] * (n.links.efficiency[heat] * n.links.c_b[electric].values) - p.loc[:, electric] * n.links.efficiency[electric] ) @@ -580,18 +581,24 @@ def extra_functionality(n, snapshots): """ opts = n.opts config = n.config - if "BAU" in opts and n.generators.p_nom_extendable.any(): + constraints = config["solving"].get("constraints", {}) + if ("BAU" in opts or constraints.get("BAU",False)) and n.generators.p_nom_extendable.any(): add_BAU_constraints(n, config) - if "SAFE" in opts and n.generators.p_nom_extendable.any(): + if ("SAFE" in opts or constraints.get("SAFE",False)) and n.generators.p_nom_extendable.any(): add_SAFE_constraints(n, config) - if "CCL" in opts and n.generators.p_nom_extendable.any(): + if ("CCL" in opts or constraints.get("CCL",False)) and n.generators.p_nom_extendable.any(): add_CCL_constraints(n, config) + reserve = config["electricity"].get("operational_reserve", {}) if reserve.get("activate"): add_operational_reserve_margin(n, snapshots, config) - for o in opts: - if "EQ" in o: - add_EQ_constraints(n, o) + + EQ_config = constraints.get("EQ",False) + EQ_wildcard = get_opt(opts, r"^EQ+[0-9]*\.?[0-9]+(c|)") + EQ_o = EQ_wildcard or EQ_config + if EQ_o: + add_EQ_constraints(n, EQ_o) + add_battery_constraints(n) add_pipe_retrofit_constraint(n) From 1e87cf0eebd447d98676f361829bd05b898013dc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 27 Sep 2023 06:22:55 +0000 Subject: [PATCH 10/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- config/config.default.yaml | 2 +- scripts/solve_network.py | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 620c04af..76490e6b 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -622,7 +622,7 @@ solving: transmission_losses: 0 linearized_unit_commitment: true horizon: 365 - + constraints: CCL: false EQ: false diff --git a/scripts/solve_network.py b/scripts/solve_network.py index c1c170ce..ca542bba 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -33,7 +33,7 @@ import numpy as np import pandas as pd import pypsa import xarray as xr -from _helpers import configure_logging, update_config_with_sector_opts, get_opt +from _helpers import configure_logging, get_opt, update_config_with_sector_opts logger = logging.getLogger(__name__) pypsa.pf.logger.setLevel(logging.WARNING) @@ -542,7 +542,6 @@ def add_chp_constraints(n): # back-pressure if not electric.empty: lhs = ( - p.loc[:, heat] * (n.links.efficiency[heat] * n.links.c_b[electric].values) - p.loc[:, electric] * n.links.efficiency[electric] ) @@ -582,18 +581,24 @@ def extra_functionality(n, snapshots): opts = n.opts config = n.config constraints = config["solving"].get("constraints", {}) - if ("BAU" in opts or constraints.get("BAU",False)) and n.generators.p_nom_extendable.any(): + if ( + "BAU" in opts or constraints.get("BAU", False) + ) and n.generators.p_nom_extendable.any(): add_BAU_constraints(n, config) - if ("SAFE" in opts or constraints.get("SAFE",False)) and n.generators.p_nom_extendable.any(): + if ( + "SAFE" in opts or constraints.get("SAFE", False) + ) and n.generators.p_nom_extendable.any(): add_SAFE_constraints(n, config) - if ("CCL" in opts or constraints.get("CCL",False)) and n.generators.p_nom_extendable.any(): + if ( + "CCL" in opts or constraints.get("CCL", False) + ) and n.generators.p_nom_extendable.any(): add_CCL_constraints(n, config) reserve = config["electricity"].get("operational_reserve", {}) if reserve.get("activate"): add_operational_reserve_margin(n, snapshots, config) - EQ_config = constraints.get("EQ",False) + EQ_config = constraints.get("EQ", False) EQ_wildcard = get_opt(opts, r"^EQ+[0-9]*\.?[0-9]+(c|)") EQ_o = EQ_wildcard or EQ_config if EQ_o: From a3a7e19b07a51ca2c0a9a8f8de4f9573a6194647 Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Thu, 28 Sep 2023 21:11:22 +0200 Subject: [PATCH 11/15] add finishing touches --- doc/configtables/snapshots.csv | 2 +- doc/configtables/solving.csv | 2 +- scripts/_helpers.py | 12 ++++++++++++ scripts/prepare_network.py | 18 ++---------------- scripts/solve_network.py | 4 ++-- 5 files changed, 18 insertions(+), 20 deletions(-) diff --git a/doc/configtables/snapshots.csv b/doc/configtables/snapshots.csv index 769e6cb0..4a3e1212 100644 --- a/doc/configtables/snapshots.csv +++ b/doc/configtables/snapshots.csv @@ -3,4 +3,4 @@ start,--,str or datetime-like; e.g. YYYY-MM-DD,Left bound of date range end,--,str or datetime-like; e.g. YYYY-MM-DD,Right bound of date range inclusive,--,"One of {'neither', 'both', ‘left’, ‘right’}","Make the time interval closed to the ``left``, ``right``, or both sides ``both`` or neither side ``None``." resolution ,--,"{false,``nH``; i.e. ``2H``-``6H``}",Resample the time-resolution by averaging over every ``n`` snapshots -segmentation,--,"{false,``nSEG``; e.g. ``4380SEG``}","Apply time series segmentation with `tsam `_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load." +segmentation,--,"{false,``n``; e.g. ``4380``}","Apply time series segmentation with `tsam `_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load." diff --git a/doc/configtables/solving.csv b/doc/configtables/solving.csv index 344bf73f..940deba9 100644 --- a/doc/configtables/solving.csv +++ b/doc/configtables/solving.csv @@ -14,7 +14,7 @@ options,,, -- horizon,--,int,Number of snapshots to consider in each iteration. Defaults to 100. constraints ,,, -- CCL,bool,"{'true','false'}",Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``. --- EQ,bool/string,"{'false',`EQn(c| )``; i.e. ``EQ0.5``-``EQ0.7c``}",Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption. +-- EQ,bool/string,"{'false',`n(c| )``; i.e. ``0.5``-``0.7c``}",Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption. -- BAU,bool,"{'true','false'}",Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities`` -- SAFE,bool,"{'true','false'}",Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network. solver,,, diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 01349e08..559997ad 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -36,6 +36,18 @@ def get_opt(opts, expr, flags=None): return match.group(0) return None +def find_opt(opts, expr): + """ + Return if available the float after the expression. + """ + for o in opts: + if expr in o: + m = re.findall("[0-9]*\.?[0-9]+$", o) + if len(m) > 0: + return True, float(m[0]) + else: + return True, None + return False, None # Define a context manager to temporarily mute print statements @contextlib.contextmanager diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 2a15cb0c..c91097ba 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -63,7 +63,7 @@ import re import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging, get_opt +from _helpers import configure_logging, get_opt, find_opt from add_electricity import load_costs, update_transmission_costs from pypsa.descriptors import expand_series @@ -72,20 +72,6 @@ idx = pd.IndexSlice logger = logging.getLogger(__name__) -def find_opt(opts, expr): - """ - Return if available the float after the expression. - """ - for o in opts: - if expr in o: - m = re.findall("[0-9]*\.?[0-9]+$", o) - if len(m) > 0: - return True, float(m[0]) - else: - return True, None - return False, None - - def add_co2limit(n, co2limit, Nyears=1.0): n.add( "GlobalConstraint", @@ -320,7 +306,7 @@ if __name__ == "__main__": # segments with package tsam time_seg_config = snakemake.params.snapshots.get("segmentation", False) - time_seg_wildcard = get_opt(opts, r"^\d+seg$") + time_seg_wildcard = get_opt(opts, r"^\d+seg$")[:-3] time_seg = time_seg_wildcard or time_seg_config if time_seg: solver_name = snakemake.config["solving"]["solver"]["name"] diff --git a/scripts/solve_network.py b/scripts/solve_network.py index ca542bba..2335c86d 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -598,8 +598,8 @@ def extra_functionality(n, snapshots): if reserve.get("activate"): add_operational_reserve_margin(n, snapshots, config) - EQ_config = constraints.get("EQ", False) - EQ_wildcard = get_opt(opts, r"^EQ+[0-9]*\.?[0-9]+(c|)") + EQ_config = constraints.get("EQ",False) + EQ_wildcard = get_opt(opts, r"^EQ+[0-9]*\.?[0-9]+(c|)")[2:] EQ_o = EQ_wildcard or EQ_config if EQ_o: add_EQ_constraints(n, EQ_o) From d75b0ae8abfea19c1da69085af58003a653ac547 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 28 Sep 2023 19:11:51 +0000 Subject: [PATCH 12/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/_helpers.py | 2 ++ scripts/prepare_network.py | 2 +- scripts/solve_network.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 559997ad..a32e8f05 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -36,6 +36,7 @@ def get_opt(opts, expr, flags=None): return match.group(0) return None + def find_opt(opts, expr): """ Return if available the float after the expression. @@ -49,6 +50,7 @@ def find_opt(opts, expr): return True, None return False, None + # Define a context manager to temporarily mute print statements @contextlib.contextmanager def mute_print(): diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index c91097ba..7ddf805b 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -63,7 +63,7 @@ import re import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging, get_opt, find_opt +from _helpers import configure_logging, find_opt, get_opt from add_electricity import load_costs, update_transmission_costs from pypsa.descriptors import expand_series diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 2335c86d..08bb51d5 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -598,7 +598,7 @@ def extra_functionality(n, snapshots): if reserve.get("activate"): add_operational_reserve_margin(n, snapshots, config) - EQ_config = constraints.get("EQ",False) + EQ_config = constraints.get("EQ", False) EQ_wildcard = get_opt(opts, r"^EQ+[0-9]*\.?[0-9]+(c|)")[2:] EQ_o = EQ_wildcard or EQ_config if EQ_o: From 4ac664b8718d36c87d655d523d8f6263d8c7e358 Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Fri, 29 Sep 2023 21:24:29 +0200 Subject: [PATCH 13/15] use .replace to fix type error --- scripts/prepare_network.py | 4 ++-- scripts/solve_network.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 7ddf805b..59668453 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -306,11 +306,11 @@ if __name__ == "__main__": # segments with package tsam time_seg_config = snakemake.params.snapshots.get("segmentation", False) - time_seg_wildcard = get_opt(opts, r"^\d+seg$")[:-3] + time_seg_wildcard = get_opt(opts, r"^\d+seg$") time_seg = time_seg_wildcard or time_seg_config if time_seg: solver_name = snakemake.config["solving"]["solver"]["name"] - n = apply_time_segmentation(n, time_seg, solver_name) + n = apply_time_segmentation(n, time_seg.replace("seg",""), solver_name) Co2L_config = snakemake.params.co2limit_enable Co2L_wildcard, co2limit_wildcard = find_opt(opts, "Co2L") diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 08bb51d5..963a85a9 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -599,10 +599,10 @@ def extra_functionality(n, snapshots): add_operational_reserve_margin(n, snapshots, config) EQ_config = constraints.get("EQ", False) - EQ_wildcard = get_opt(opts, r"^EQ+[0-9]*\.?[0-9]+(c|)")[2:] + EQ_wildcard = get_opt(opts, r"^EQ+[0-9]*\.?[0-9]+(c|)") EQ_o = EQ_wildcard or EQ_config if EQ_o: - add_EQ_constraints(n, EQ_o) + add_EQ_constraints(n, EQ_o.replace("EQ","")) add_battery_constraints(n) add_pipe_retrofit_constraint(n) From d2f8dc25fd3fa7497d3c3c8b6a7d820f593fc09b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 19:24:53 +0000 Subject: [PATCH 14/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/prepare_network.py | 2 +- scripts/solve_network.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 59668453..f2c6556e 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -310,7 +310,7 @@ if __name__ == "__main__": time_seg = time_seg_wildcard or time_seg_config if time_seg: solver_name = snakemake.config["solving"]["solver"]["name"] - n = apply_time_segmentation(n, time_seg.replace("seg",""), solver_name) + n = apply_time_segmentation(n, time_seg.replace("seg", ""), solver_name) Co2L_config = snakemake.params.co2limit_enable Co2L_wildcard, co2limit_wildcard = find_opt(opts, "Co2L") diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 963a85a9..012e440b 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -602,7 +602,7 @@ def extra_functionality(n, snapshots): EQ_wildcard = get_opt(opts, r"^EQ+[0-9]*\.?[0-9]+(c|)") EQ_o = EQ_wildcard or EQ_config if EQ_o: - add_EQ_constraints(n, EQ_o.replace("EQ","")) + add_EQ_constraints(n, EQ_o.replace("EQ", "")) add_battery_constraints(n) add_pipe_retrofit_constraint(n) From 558763e40ce651062449f1b113d3e306bd4ccce1 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 5 Jan 2024 14:32:37 +0100 Subject: [PATCH 15/15] add release notes --- config/config.default.yaml | 1 - doc/release_notes.rst | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index a5e49539..b9e8dc91 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -90,7 +90,6 @@ co2_budget: # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#electricity electricity: - voltages: [220., 300., 380.] voltages: [220., 300., 380., 500., 750.] gaslimit_enable: false gaslimit: false diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 88f05854..7a045866 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,11 @@ Release Notes Upcoming Release ================ +* More wildcard options now have a corresponding config entry. If the wildcard + is given, then its value is used. If the wildcard is not given but the options + in config are enabled, then the value from config is used. If neither is + given, the options are skipped. + * Distinguish between stored and sequestered CO2. Stored CO2 is stored overground in tanks and can be used for CCU (e.g. methanolisation). Sequestered CO2 is stored underground and can no longer be used for CCU. This