diff --git a/data/override_component_attrs/buses.csv b/data/override_component_attrs/buses.csv deleted file mode 100644 index 7581e328..00000000 --- a/data/override_component_attrs/buses.csv +++ /dev/null @@ -1,3 +0,0 @@ -attribute,type,unit,default,description,status -location,string,n/a,n/a,Reference to original electricity bus,Input (optional) -unit,string,n/a,MWh,Unit of the bus (descriptive only), Input (optional) diff --git a/data/override_component_attrs/generators.csv b/data/override_component_attrs/generators.csv deleted file mode 100644 index 4f214396..00000000 --- a/data/override_component_attrs/generators.csv +++ /dev/null @@ -1,4 +0,0 @@ -attribute,type,unit,default,description,status -carrier,string,n/a,n/a,carrier,Input (optional) -lifetime,float,years,inf,lifetime,Input (optional) -build_year,int,year ,0,build year,Input (optional) diff --git a/data/override_component_attrs/links.csv b/data/override_component_attrs/links.csv deleted file mode 100644 index 0fc2747a..00000000 --- a/data/override_component_attrs/links.csv +++ /dev/null @@ -1,13 +0,0 @@ -attribute,type,unit,default,description,status -bus2,string,n/a,n/a,2nd bus,Input (optional) -bus3,string,n/a,n/a,3rd bus,Input (optional) -bus4,string,n/a,n/a,4th bus,Input (optional) -efficiency2,static or series,per unit,1,2nd bus efficiency,Input (optional) -efficiency3,static or series,per unit,1,3rd bus efficiency,Input (optional) -efficiency4,static or series,per unit,1,4th bus efficiency,Input (optional) -p2,series,MW,0,2nd bus output,Output -p3,series,MW,0,3rd bus output,Output -p4,series,MW,0,4th bus output,Output -carrier,string,n/a,n/a,carrier,Input (optional) -lifetime,float,years,inf,lifetime,Input (optional) -build_year,int,year ,0,build year,Input (optional) diff --git a/data/override_component_attrs/loads.csv b/data/override_component_attrs/loads.csv deleted file mode 100644 index 10bb5b4f..00000000 --- a/data/override_component_attrs/loads.csv +++ /dev/null @@ -1,2 +0,0 @@ -attribute,type,unit,default,description,status -carrier,string,n/a,n/a,carrier,Input (optional) diff --git a/data/override_component_attrs/stores.csv b/data/override_component_attrs/stores.csv deleted file mode 100644 index 4f214396..00000000 --- a/data/override_component_attrs/stores.csv +++ /dev/null @@ -1,4 +0,0 @@ -attribute,type,unit,default,description,status -carrier,string,n/a,n/a,carrier,Input (optional) -lifetime,float,years,inf,lifetime,Input (optional) -build_year,int,year ,0,build year,Input (optional) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 4c7219e7..fd23e888 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -40,11 +40,15 @@ Upcoming Release e.g. by setting ``solving: options: transmission_losses: 2`` for an approximation with two tangents. +* Handling networks with links with multiple inputs/outputs no longer requires + to override component attributes. + * Added configuration option ``enable: retrieve:`` to control whether data retrieval rules from snakemake are enabled or not. Th default setting ``auto`` will automatically detect and enable/disable the rules based on internet connectivity. + PyPSA-Eur 0.8.0 (18th March 2023) ================================= diff --git a/envs/environment.yaml b/envs/environment.yaml index 1a2359dc..8409adb0 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -10,7 +10,6 @@ dependencies: - python>=3.8 - pip -# - pypsa>=0.23 - atlite>=0.2.9 - dask diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 1c3507fd..86f8bab2 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -724,7 +724,6 @@ rule prepare_sector_network: **build_biomass_transport_costs_output, **gas_infrastructure, **build_sequestration_potentials_output, - overrides="data/override_component_attrs", network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", energy_totals_name=RESOURCES + "energy_totals.csv", eurostat=input_eurostat, diff --git a/rules/postprocess.smk b/rules/postprocess.smk index ac80cd10..105318f3 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -13,7 +13,6 @@ rule plot_network: foresight=config["foresight"], plotting=config["plotting"], input: - overrides="data/override_component_attrs", network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -76,7 +75,6 @@ rule make_summary: scenario=config["scenario"], RDIR=RDIR, input: - overrides="data/override_component_attrs", networks=expand( RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 086375c2..ba2787a1 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -10,7 +10,6 @@ rule add_existing_baseyear: existing_capacities=config["existing_capacities"], costs=config["costs"], input: - overrides="data/override_component_attrs", network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", powerplants=RESOURCES + "powerplants.csv", @@ -52,7 +51,6 @@ rule add_brownfield: H2_retrofit_capacity_per_CH4=config["sector"]["H2_retrofit_capacity_per_CH4"], threshold_capacity=config["existing_capacities"]["threshold_capacity"], input: - overrides="data/override_component_attrs", network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", network_p=solved_previous_horizon, #solved network at previous time step @@ -91,7 +89,6 @@ rule solve_sector_network_myopic: "co2_sequestration_potential", 200 ), input: - overrides="data/override_component_attrs", network=RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", costs="data/costs_{planning_horizons}.csv", diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index fc430f4d..d0313f30 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -12,7 +12,6 @@ rule solve_sector_network: "co2_sequestration_potential", 200 ), input: - overrides="data/override_component_attrs", network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", costs="data/costs_{}.csv".format(config["costs"]["year"]), diff --git a/scripts/_helpers.py b/scripts/_helpers.py index cd702950..fc7bc9e0 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -72,92 +72,6 @@ def configure_logging(snakemake, skip_handlers=False): logging.basicConfig(**kwargs) -def load_network(import_name=None, custom_components=None): - """ - Helper for importing a pypsa.Network with additional custom components. - - Parameters - ---------- - import_name : str - As in pypsa.Network(import_name) - custom_components : dict - Dictionary listing custom components. - For using ``snakemake.params['override_components']`` - in ``config/config.yaml`` define: - - .. code:: yaml - - override_components: - ShadowPrice: - component: ["shadow_prices","Shadow price for a global constraint.",np.nan] - attributes: - name: ["string","n/a","n/a","Unique name","Input (required)"] - value: ["float","n/a",0.,"shadow value","Output"] - - Returns - ------- - pypsa.Network - """ - import pypsa - from pypsa.descriptors import Dict - - override_components = None - override_component_attrs = None - - if custom_components is not None: - override_components = pypsa.components.components.copy() - override_component_attrs = Dict( - {k: v.copy() for k, v in pypsa.components.component_attrs.items()} - ) - for k, v in custom_components.items(): - override_components.loc[k] = v["component"] - override_component_attrs[k] = pd.DataFrame( - columns=["type", "unit", "default", "description", "status"] - ) - for attr, val in v["attributes"].items(): - override_component_attrs[k].loc[attr] = val - - return pypsa.Network( - import_name=import_name, - override_components=override_components, - override_component_attrs=override_component_attrs, - ) - - -def load_network_for_plots(fn, tech_costs, config, combine_hydro_ps=True): - import pypsa - from add_electricity import load_costs, update_transmission_costs - - n = pypsa.Network(fn) - - n.loads["carrier"] = n.loads.bus.map(n.buses.carrier) + " load" - n.stores["carrier"] = n.stores.bus.map(n.buses.carrier) - - n.links["carrier"] = ( - n.links.bus0.map(n.buses.carrier) + "-" + n.links.bus1.map(n.buses.carrier) - ) - n.lines["carrier"] = "AC line" - n.transformers["carrier"] = "AC transformer" - - n.lines["s_nom"] = n.lines["s_nom_min"] - n.links["p_nom"] = n.links["p_nom_min"] - - if combine_hydro_ps: - n.storage_units.loc[ - n.storage_units.carrier.isin({"PHS", "hydro"}), "carrier" - ] = "hydro+PHS" - - # if the carrier was not set on the heat storage units - # bus_carrier = n.storage_units.bus.map(n.buses.carrier) - # n.storage_units.loc[bus_carrier == "heat","carrier"] = "water tanks" - - Nyears = n.snapshot_weightings.objective.sum() / 8760.0 - costs = load_costs(tech_costs, config["costs"], config["electricity"], Nyears) - update_transmission_costs(n, costs) - - return n - - def update_p_nom_max(n): # if extendable carriers (solar/onwind/...) have capacity >= 0, # e.g. existing assets from the OPSD project are included to the network, @@ -367,34 +281,6 @@ def mock_snakemake(rulename, configfiles=[], **wildcards): return snakemake -def override_component_attrs(directory): - """ - Tell PyPSA that links can have multiple outputs by overriding the - component_attrs. This can be done for as many buses as you need with format - busi for i = 2,3,4,5,.... See https://pypsa.org/doc/components.html#link- - with-multiple-outputs-or-inputs. - - Parameters - ---------- - directory : string - Folder where component attributes to override are stored - analogous to ``pypsa/component_attrs``, e.g. `links.csv`. - - Returns - ------- - Dictionary of overridden component attributes. - """ - attrs = Dict({k: v.copy() for k, v in component_attrs.items()}) - - for component, list_name in components.list_name.items(): - fn = f"{directory}/{list_name}.csv" - if os.path.isfile(fn): - overrides = pd.read_csv(fn, index_col=0, na_values="n/a") - attrs[component] = overrides.combine_first(attrs[component]) - - return attrs - - def generate_periodic_profiles(dt_index, nodes, weekly_profile, localize=None): """ Give a 24*7 long list of weekly hourly profiles, generate this for each diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index b33d2f19..597792c0 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -16,7 +16,7 @@ idx = pd.IndexSlice import numpy as np import pypsa -from _helpers import override_component_attrs, update_config_with_sector_opts +from _helpers import update_config_with_sector_opts from add_existing_baseyear import add_build_year_to_new_assets @@ -147,12 +147,11 @@ if __name__ == "__main__": year = int(snakemake.wildcards.planning_horizons) - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) + n = pypsa.Network(snakemake.input.network) add_build_year_to_new_assets(n, year) - n_p = pypsa.Network(snakemake.input.network_p, override_component_attrs=overrides) + n_p = pypsa.Network(snakemake.input.network_p) add_brownfield(n, n_p, year) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index ce857742..24567477 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -424,6 +424,13 @@ def attach_conventional_generators( add_missing_carriers(n, carriers) add_co2_emissions(n, costs, carriers) + # Replace carrier "natural gas" with the respective technology (OCGT or + # CCGT) to align with PyPSA names of "carriers" and avoid filtering "natural + # gas" powerplants in ppl.query("carrier in @carriers") + ppl.loc[ppl["carrier"] == "natural gas", "carrier"] = ppl.loc[ + ppl["carrier"] == "natural gas", "technology" + ] + ppl = ( ppl.query("carrier in @carriers") .join(costs, on="carrier", rsuffix="_r") diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index f80bffb8..b2c534e6 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -21,7 +21,7 @@ import country_converter as coco import numpy as np import pypsa import xarray as xr -from _helpers import override_component_attrs, update_config_with_sector_opts +from _helpers import update_config_with_sector_opts from add_electricity import sanitize_carriers from prepare_sector_network import cluster_heat_buses, define_spatial, prepare_costs @@ -129,10 +129,14 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas "Oil": "oil", "OCGT": "OCGT", "CCGT": "CCGT", - "Natural Gas": "gas", "Bioenergy": "urban central solid biomass CHP", } + # Replace Fueltype "Natural Gas" with the respective technology (OCGT or CCGT) + df_agg.loc[df_agg["Fueltype"] == "Natural Gas", "Fueltype"] = df_agg.loc[ + df_agg["Fueltype"] == "Natural Gas", "Technology" + ] + fueltype_to_drop = [ "Hydro", "Wind", @@ -601,12 +605,13 @@ if __name__ == "__main__": snakemake = mock_snakemake( "add_existing_baseyear", + configfiles="config/test/config.myopic.yaml", simpl="", - clusters="45", - ll="v1.0", + clusters="5", + ll="v1.5", opts="", - sector_opts="8760H-T-H-B-I-A-solar+p3-dist1", - planning_horizons=2020, + sector_opts="24H-T-H-B-I-A-solar+p3-dist1", + planning_horizons=2030, ) logging.basicConfig(level=snakemake.config["logging"]["level"]) @@ -618,8 +623,8 @@ if __name__ == "__main__": baseyear = snakemake.params.baseyear - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) + n = pypsa.Network(snakemake.input.network) + # define spatial resolution of carriers spatial = define_spatial(n.buses[n.buses.carrier == "AC"].index, options) add_build_year_to_new_assets(n, baseyear) diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index f25f111b..cbe94505 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -98,13 +98,15 @@ def add_custom_powerplants(ppl, custom_powerplants, custom_ppl_query=False): def replace_natural_gas_technology(df): - mapping = {"Steam Turbine": "OCGT", "Combustion Engine": "OCGT"} - tech = df.Technology.replace(mapping).fillna("OCGT") - return df.Technology.where(df.Fueltype != "Natural Gas", tech) + mapping = {"Steam Turbine": "CCGT", "Combustion Engine": "OCGT"} + tech = df.Technology.replace(mapping).fillna("CCGT") + return df.Technology.mask(df.Fueltype == "Natural Gas", tech) def replace_natural_gas_fueltype(df): - return df.Fueltype.where(df.Fueltype != "Natural Gas", df.Technology) + return df.Fueltype.mask( + (df.Technology == "OCGT") | (df.Technology == "CCGT"), "Natural Gas" + ) if __name__ == "__main__": diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 00fca2fd..56ee98c9 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -16,7 +16,6 @@ import sys import numpy as np import pandas as pd import pypsa -from _helpers import override_component_attrs from prepare_sector_network import prepare_costs idx = pd.IndexSlice @@ -300,9 +299,9 @@ def calculate_energy(n, label, energy): ) # remove values where bus is missing (bug in nomopyomo) no_bus = c.df.index[c.df["bus" + port] == ""] - totals.loc[no_bus] = n.component_attrs[c.name].loc[ - "p" + port, "default" - ] + totals.loc[no_bus] = float( + n.component_attrs[c.name].loc["p" + port, "default"] + ) c_energies -= totals.groupby(c.df.carrier).sum() c_energies = pd.concat([c_energies], keys=[c.list_name]) @@ -659,8 +658,7 @@ def make_summaries(networks_dict): for label, filename in networks_dict.items(): logger.info(f"Make summary for scenario {label}, using {filename}") - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network(filename, override_component_attrs=overrides) + n = pypsa.Network(filename) assign_carriers(n) assign_locations(n) diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 184d86f0..ae1d0e0a 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -20,7 +20,6 @@ import geopandas as gpd import matplotlib.pyplot as plt import pandas as pd import pypsa -from _helpers import override_component_attrs from make_summary import assign_carriers from plot_summary import preferred_order, rename_techs from pypsa.plot import add_legend_circles, add_legend_lines, add_legend_patches @@ -930,8 +929,7 @@ if __name__ == "__main__": logging.basicConfig(level=snakemake.config["logging"]["level"]) - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) + n = pypsa.Network(snakemake.input.network) regions = gpd.read_file(snakemake.input.regions).set_index("name") diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index da6e693a..e0afbcd8 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -17,11 +17,7 @@ import numpy as np import pandas as pd import pypsa import xarray as xr -from _helpers import ( - generate_periodic_profiles, - override_component_attrs, - update_config_with_sector_opts, -) +from _helpers import generate_periodic_profiles, update_config_with_sector_opts from add_electricity import calculate_annuity, sanitize_carriers from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2 from networkx.algorithms import complement @@ -3279,8 +3275,7 @@ if __name__ == "__main__": investment_year = int(snakemake.wildcards.planning_horizons[-4:]) - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) + n = pypsa.Network(snakemake.input.network) pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0) nhours = n.snapshot_weightings.generators.sum() diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 98b65737..c4432cda 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -33,11 +33,7 @@ import numpy as np import pandas as pd import pypsa import xarray as xr -from _helpers import ( - configure_logging, - override_component_attrs, - update_config_with_sector_opts, -) +from _helpers import configure_logging, update_config_with_sector_opts logger = logging.getLogger(__name__) pypsa.pf.logger.setLevel(logging.WARNING) @@ -674,11 +670,7 @@ if __name__ == "__main__": np.random.seed(solve_opts.get("seed", 123)) - if "overrides" in snakemake.input.keys(): - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) - else: - n = pypsa.Network(snakemake.input.network) + n = pypsa.Network(snakemake.input.network) n = prepare_network( n, diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 37e853e5..1a3855a9 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -11,11 +11,7 @@ import logging import numpy as np import pypsa -from _helpers import ( - configure_logging, - override_component_attrs, - update_config_with_sector_opts, -) +from _helpers import configure_logging, update_config_with_sector_opts from solve_network import prepare_network, solve_network logger = logging.getLogger(__name__) @@ -45,11 +41,7 @@ if __name__ == "__main__": np.random.seed(solve_opts.get("seed", 123)) - if "overrides" in snakemake.input: - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) - else: - n = pypsa.Network(snakemake.input.network) + n = pypsa.Network(snakemake.input.network) n.optimize.fix_optimal_capacities() n = prepare_network(n, solve_opts, config=snakemake.config)