From 45cac01ea3c49529c4b68714e783bedb60565f00 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 14 Jun 2023 13:28:20 +0200 Subject: [PATCH] adjust param accessing in cluster, simplify and build_elec script --- rules/build_electricity.smk | 24 ++--- scripts/add_electricity.py | 40 ++++----- scripts/build_renewable_profiles.py | 2 +- scripts/cluster_network.py | 56 +++++------- scripts/simplify_network.py | 131 ++++++++++++---------------- 5 files changed, 109 insertions(+), 144 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 34276ba4..fd66cc04 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -316,16 +316,14 @@ rule add_electricity: rule simplify_network: params: - clustering=config["clustering"], - max_hours=config["electricity"]["max_hours"], + simplify_network=config["clustering"]["simplify_network"], + aggregation_strategies=config["clustering"].get("aggregation_strategies", {}), + focus_weights=config.get("focus_weights", None), + renewable_carriers=config["electricity"]["renewable_carriers"], costs=config["costs"], - renewable=config["renewable"], + max_hours=config["electricity"]["max_hours"], length_factor=config["lines"]["length_factor"], p_max_pu=config["links"].get("p_max_pu", 1.0), - exclude_carriers=config["clustering"]["simplify_network"].get( - "exclude_carriers", [] - ), - focus_weights=config.get("focus_weights", None), solver_name=config["solving"]["solver"]["name"], input: network=RESOURCES + "networks/elec.nc", @@ -353,14 +351,16 @@ rule simplify_network: rule cluster_network: params: - solver_name=config["solving"]["solver"]["name"], - max_hours=config["electricity"]["max_hours"], + cluster_network=config["clustering"]["cluster_network"], + aggregation_strategies=config["clustering"].get("aggregation_strategies", {}), + custom_busmap=config["enable"].get("custom_busmap", False), + focus_weights=config.get("focus_weights", None), + renewable_carriers=config["electricity"]["renewable_carriers"], conventional_carriers=config["electricity"].get("conventional_carriers", []), costs=config["costs"], + max_hours=config["electricity"]["max_hours"], length_factor=config["lines"]["length_factor"], - renewable=config["renewable"], - clustering=config["clustering"], - custom_busmap=config["enable"].get("custom_busmap", False), + solver_name=config["solving"]["solver"]["name"], input: network=RESOURCES + "networks/elec_s{simpl}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson", diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index dc5fb7cc..34147af3 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -726,23 +726,23 @@ if __name__ == "__main__": costs = load_costs( snakemake.input.tech_costs, - snakemake.params["costs"], - snakemake.params["electricity"]["max_hours"], + snakemake.params.costs, + snakemake.params.electricity["max_hours"], Nyears, ) ppl = load_powerplants(snakemake.input.powerplants) - if "renewable_carriers" in snakemake.params["electricity"]: - renewable_carriers = set(snakemake.params["electricity"]["renewable_carriers"]) + if "renewable_carriers" in snakemake.params.electricity: + renewable_carriers = set(snakemake.params.electricity["renewable_carriers"]) else: logger.warning( "Missing key `renewable_carriers` under config entry `electricity`. " "In future versions, this will raise an error. " "Falling back to carriers listed under `renewable`." ) - renewable_carriers = snakemake.params["renewable"] + renewable_carriers = snakemake.params.renewable - extendable_carriers = snakemake.params["electricity"]["extendable_carriers"] + extendable_carriers = snakemake.params.electricity["extendable_carriers"] if not (set(renewable_carriers) & set(extendable_carriers["Generator"])): logger.warning( "No renewables found in config entry `extendable_carriers`. " @@ -750,18 +750,18 @@ if __name__ == "__main__": "Falling back to all renewables." ) - conventional_carriers = snakemake.params["electricity"]["conventional_carriers"] + conventional_carriers = snakemake.params.electricity["conventional_carriers"] attach_load( n, snakemake.input.regions, snakemake.input.load, snakemake.input.nuts3_shapes, - snakemake.params["countries"], - snakemake.params["scaling_factor"], + snakemake.params.countries, + snakemake.params.scaling_factor, ) - update_transmission_costs(n, costs, snakemake.params["length_factor"]) + update_transmission_costs(n, costs, snakemake.params.length_factor) conventional_inputs = { k: v for k, v in snakemake.input.items() if k.startswith("conventional_") @@ -772,7 +772,7 @@ if __name__ == "__main__": ppl, conventional_carriers, extendable_carriers, - snakemake.params["conventional"], + snakemake.params.conventional, conventional_inputs, ) @@ -782,11 +782,11 @@ if __name__ == "__main__": snakemake.input, renewable_carriers, extendable_carriers, - snakemake.params["length_factor"], + snakemake.params.length_factor, ) if "hydro" in renewable_carriers: - para = snakemake.params["renewable"]["hydro"] + para = snakemake.params.renewable["hydro"] attach_hydro( n, costs, @@ -797,7 +797,7 @@ if __name__ == "__main__": **para, ) - if "estimate_renewable_capacities" not in snakemake.params["electricity"]: + if "estimate_renewable_capacities" not in snakemake.params.electricity: logger.warning( "Missing key `estimate_renewable_capacities` under config entry `electricity`. " "In future versions, this will raise an error. " @@ -805,18 +805,18 @@ if __name__ == "__main__": ) if ( "estimate_renewable_capacities_from_capacity_stats" - in snakemake.params["electricity"] + in snakemake.params.electricity ): estimate_renewable_caps = { "enable": True, - **snakemake.params["electricity"][ + **snakemake.params.electricity[ "estimate_renewable_capacities_from_capacity_stats" ], } else: estimate_renewable_caps = {"enable": False} else: - estimate_renewable_caps = snakemake.params["electricity"][ + estimate_renewable_caps = snakemake.params.electricity[ "estimate_renewable_capacities" ] if "enable" not in estimate_renewable_caps: @@ -832,18 +832,18 @@ if __name__ == "__main__": "Falling back to whether `renewable_capacities_from_opsd` is non-empty." ) from_opsd = bool( - snakemake.params["electricity"].get("renewable_capacities_from_opsd", False) + snakemake.params.electricity.get("renewable_capacities_from_opsd", False) ) estimate_renewable_caps["from_opsd"] = from_opsd if estimate_renewable_caps["enable"]: if estimate_renewable_caps["from_opsd"]: - tech_map = snakemake.params["electricity"]["estimate_renewable_capacities"][ + tech_map = snakemake.params.electricity["estimate_renewable_capacities"][ "technology_mapping" ] attach_OPSD_renewables(n, tech_map) estimate_renewable_capacities( - n, snakemake.params["electricity"], snakemake.params["countries"] + n, snakemake.params.electricity, snakemake.params.countries ) update_p_nom_max(n) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 91463c23..9e90d42b 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -309,7 +309,7 @@ if __name__ == "__main__": p_nom_max = capacities / max_cap_factor else: raise AssertionError( - 'params key `potential` should be one of "simple" ' + 'Config key `potential` should be one of "simple" ' f'(default) or "conservative", not "{p_nom_max_meth}"' ) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index db0fb2c2..52685af2 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -186,7 +186,7 @@ def get_feature_for_hac(n, buses_i=None, feature=None): if "offwind" in carriers: carriers.remove("offwind") carriers = np.append( - carriers, network.generators.carrier.filter(like="offwind").unique() + carriers, n.generators.carrier.filter(like="offwind").unique() ) if feature.split("-")[1] == "cap": @@ -463,26 +463,18 @@ if __name__ == "__main__": snakemake = mock_snakemake("cluster_network", simpl="", clusters="5") configure_logging(snakemake) + params = snakemake.params + solver_name = snakemake.config["solving"]["solver"]["name"] + n = pypsa.Network(snakemake.input.network) - focus_weights = snakemake.config.get("focus_weights", None) - - renewable_carriers = pd.Index( - [ - tech - for tech in n.generators.carrier.unique() - if tech in snakemake.params["renewable"] - ] - ) - - exclude_carriers = snakemake.params["clustering"]["cluster_network"].get( - "exclude_carriers", [] - ) + exclude_carriers = params.cluster_network["exclude_carriers"] aggregate_carriers = set(n.generators.carrier) - set(exclude_carriers) if snakemake.wildcards.clusters.endswith("m"): n_clusters = int(snakemake.wildcards.clusters[:-1]) - conventional = set(snakemake.params["conventional_carriers"]) - aggregate_carriers = conventional.intersection(aggregate_carriers) + aggregate_carriers = set(params.conventional_carriers).intersection( + aggregate_carriers + ) elif snakemake.wildcards.clusters == "all": n_clusters = len(n.buses) else: @@ -496,13 +488,12 @@ if __name__ == "__main__": n, busmap, linemap, linemap, pd.Series(dtype="O") ) else: - line_length_factor = snakemake.params["length_factor"] Nyears = n.snapshot_weightings.objective.sum() / 8760 hvac_overhead_cost = load_costs( snakemake.input.tech_costs, - snakemake.params["costs"], - snakemake.params["max_hours"], + params.costs, + params.max_hours, Nyears, ).at["HVAC overhead", "capital_cost"] @@ -513,16 +504,16 @@ if __name__ == "__main__": ).all() or x.isnull().all(), "The `potential` configuration option must agree for all renewable carriers, for now!" return v - aggregation_strategies = snakemake.params["clustering"].get( - "aggregation_strategies", {} - ) # translate str entries of aggregation_strategies to pd.Series functions: aggregation_strategies = { - p: {k: getattr(pd.Series, v) for k, v in aggregation_strategies[p].items()} - for p in aggregation_strategies.keys() + p: { + k: getattr(pd.Series, v) + for k, v in params.aggregation_strategies[p].items() + } + for p in params.aggregation_strategies.keys() } - custom_busmap = snakemake.params["custom_busmap"] + custom_busmap = params.custom_busmap if custom_busmap: custom_busmap = pd.read_csv( snakemake.input.custom_busmap, index_col=0, squeeze=True @@ -530,21 +521,18 @@ if __name__ == "__main__": custom_busmap.index = custom_busmap.index.astype(str) logger.info(f"Imported custom busmap from {snakemake.input.custom_busmap}") - cluster_config = snakemake.config.get("clustering", {}).get( - "cluster_network", {} - ) clustering = clustering_for_n_clusters( n, n_clusters, custom_busmap, aggregate_carriers, - line_length_factor, - aggregation_strategies, - snakemake.params["solver_name"], - cluster_config.get("algorithm", "hac"), - cluster_config.get("feature", "solar+onwind-time"), + params.length_factor, + params.aggregation_strategies, + solver_name, + params.cluster_network["algorithm"], + params.cluster_network["feature"], hvac_overhead_cost, - focus_weights, + params.focus_weights, ) update_p_nom_max(clustering.network) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 83e932a3..79161760 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -149,17 +149,17 @@ def simplify_network_to_380(n): return n, trafo_map -def _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param): +def _prepare_connection_costs_per_link(n, costs, renewable_carriers, length_factor): if n.links.empty: return {} connection_costs_per_link = {} - for tech in renewable_param: + for tech in renewable_carriers: if tech.startswith("offwind"): connection_costs_per_link[tech] = ( n.links.length - * length_factor_param + * length_factor * ( n.links.underwater_fraction * costs.at[tech + "-connection-submarine", "capital_cost"] @@ -175,14 +175,14 @@ def _compute_connection_costs_to_bus( n, busmap, costs, - renewable_param, - length_factor_param, + renewable_carriers, + length_factor, connection_costs_per_link=None, buses=None, ): if connection_costs_per_link is None: connection_costs_per_link = _prepare_connection_costs_per_link( - n, costs, renewable_param, length_factor_param + n, costs, renewable_carriers, length_factor ) if buses is None: @@ -276,10 +276,10 @@ def _aggregate_and_move_components( def simplify_links( n, costs, - renewable_param, - length_factor_param, - p_max_pu_param, - exclude_carriers_param, + renewables, + length_factor, + p_max_pu, + exclude_carriers, output, aggregation_strategies=dict(), ): @@ -333,7 +333,7 @@ def simplify_links( busmap = n.buses.index.to_series() connection_costs_per_link = _prepare_connection_costs_per_link( - n, costs, renewable_param, length_factor_param + n, costs, renewables, length_factor ) connection_costs_to_bus = pd.DataFrame( 0.0, index=n.buses.index, columns=list(connection_costs_per_link) @@ -355,8 +355,8 @@ def simplify_links( n, busmap, costs, - renewable_param, - length_factor_param, + renewables, + length_factor, connection_costs_per_link, buses, ) @@ -378,8 +378,8 @@ def simplify_links( / lengths.sum() * n.links.loc[all_links, "underwater_fraction"] ), - p_max_pu=p_max_pu_param, - p_min_pu=-p_max_pu_param, + p_max_pu=p_max_pu, + p_min_pu=-p_max_pu, underground=False, under_construction=False, ) @@ -407,7 +407,7 @@ def simplify_links( connection_costs_to_bus, output, aggregation_strategies=aggregation_strategies, - exclude_carriers=exclude_carriers_param, + exclude_carriers=exclude_carriers, ) return n, busmap @@ -415,34 +415,29 @@ def simplify_links( def remove_stubs( n, costs, - renewable_param, - length_factor_param, - clustering_param, - exclude_carriers_param, + renewable_carriers, + length_factor, + simplify_network, output, aggregation_strategies=dict(), ): logger.info("Removing stubs") - across_borders = clustering_param["simplify_network"].get( - "remove_stubs_across_borders", True - ) + across_borders = simplify_network["remove_stubs_across_borders"] matching_attrs = [] if across_borders else ["country"] busmap = busmap_by_stubs(n, matching_attrs) connection_costs_to_bus = _compute_connection_costs_to_bus( - n, busmap, costs, renewable_param, length_factor_param + n, busmap, costs, renewable_carriers, length_factor ) - exclude_carriers = clustering_param["simplify_network"].get("exclude_carriers", []) - _aggregate_and_move_components( n, busmap, connection_costs_to_bus, output, aggregation_strategies=aggregation_strategies, - exclude_carriers=exclude_carriers, + exclude_carriers=simplify_network["exclude_carriers"], ) return n, busmap @@ -504,32 +499,23 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None): def cluster( n, n_clusters, - focus_weights_param, - renewable_param, - solver_name_param, + focus_weights, + solver_name, algorithm="hac", feature=None, aggregation_strategies=dict(), ): logger.info(f"Clustering to {n_clusters} buses") - renewable_carriers = pd.Index( - [ - tech - for tech in n.generators.carrier.unique() - if tech.split("-", 2)[0] in renewable_param - ] - ) - clustering = clustering_for_n_clusters( n, n_clusters, custom_busmap=False, aggregation_strategies=aggregation_strategies, - solver_name=solver_name_param, + solver_name=solver_name, algorithm=algorithm, feature=feature, - focus_weights=focus_weights_param, + focus_weights=focus_weights, ) return clustering.network, clustering.busmap @@ -542,77 +528,69 @@ if __name__ == "__main__": snakemake = mock_snakemake("simplify_network", simpl="") configure_logging(snakemake) - n = pypsa.Network(snakemake.input.network) + params = snakemake.params + solver_name = snakemake.config["solving"]["solver"]["name"] + + n = pypsa.Network(snakemake.input.network) + Nyears = n.snapshot_weightings.objective.sum() / 8760 - aggregation_strategies = snakemake.params["clustering"].get( - "aggregation_strategies", {} - ) # translate str entries of aggregation_strategies to pd.Series functions: aggregation_strategies = { - p: {k: getattr(pd.Series, v) for k, v in aggregation_strategies[p].items()} - for p in aggregation_strategies.keys() + p: { + k: getattr(pd.Series, v) + for k, v in params.aggregation_strategies[p].items() + } + for p in params.aggregation_strategies.keys() } n, trafo_map = simplify_network_to_380(n) - Nyears = n.snapshot_weightings.objective.sum() / 8760 - technology_costs = load_costs( snakemake.input.tech_costs, - snakemake.params["costs"], - snakemake.params["max_hours"], + params.costs, + params.max_hours, Nyears, ) n, simplify_links_map = simplify_links( n, technology_costs, - snakemake.params["renewable"], - snakemake.params["length_factor"], - snakemake.params["p_max_pu"], - snakemake.params["exclude_carriers"], + params.renewable_carriers, + params.length_factor, + params.p_max_pu, + params.simplify_network["exclude_carriers"], snakemake.output, aggregation_strategies, ) busmaps = [trafo_map, simplify_links_map] - cluster_param = snakemake.params["clustering"]["simplify_network"] - if cluster_param.get("remove_stubs", True): + if params.simplify_network["remove_stubs"]: n, stub_map = remove_stubs( n, technology_costs, - snakemake.params["renewable"], - snakemake.params["length_factor"], - snakemake.params["clustering"], - snakemake.params["exclude_carriers"], + params.renewable_carriers, + params.length_factor, + params.simplify_network, snakemake.output, aggregation_strategies=aggregation_strategies, ) busmaps.append(stub_map) - if cluster_param.get("to_substations", False): + if params.simplify_network["to_substations"]: n, substation_map = aggregate_to_substations(n, aggregation_strategies) busmaps.append(substation_map) # treatment of outliers (nodes without a profile for considered carrier): # all nodes that have no profile of the given carrier are being aggregated to closest neighbor - if ( - snakemake.config.get("clustering", {}) - .get("cluster_network", {}) - .get("algorithm", "hac") - == "hac" - or cluster_param.get("algorithm", "hac") == "hac" - ): - carriers = ( - cluster_param.get("feature", "solar+onwind-time").split("-")[0].split("+") - ) + if params.simplify_network["algorithm"] == "hac": + carriers = params.simplify_network["feature"].split("-")[0].split("+") for carrier in carriers: buses_i = list( set(n.buses.index) - set(n.generators.query("carrier == @carrier").bus) ) logger.info( - f"clustering preparaton (hac): aggregating {len(buses_i)} buses of type {carrier}." + f"clustering preparation (hac): aggregating {len(buses_i)} buses of type {carrier}." ) n, busmap_hac = aggregate_to_substations(n, aggregation_strategies, buses_i) busmaps.append(busmap_hac) @@ -621,11 +599,10 @@ if __name__ == "__main__": n, cluster_map = cluster( n, int(snakemake.wildcards.simpl), - snakemake.params["focus_weights"], - snakemake.params["renewable"], - snakemake.params["solver_name"], - cluster_param.get("algorithm", "hac"), - cluster_param.get("feature", None), + params.focus_weights, + solver_name, + params.simplify_network["algorithm"], + params.simplify_network["feature"], aggregation_strategies, ) busmaps.append(cluster_map)