Merge branch 'validation' of github.com:PyPSA/pypsa-eur into validation

Snakefile

@@ -107,6 +107,6 @@ rule sync:
     shell:
         """
         rsync -uvarh --ignore-missing-args --files-from=.sync-send . {params.cluster}
-        rsync -uvarh --no-g {params.cluster}/results results || echo "No results directory, skipping rsync"
+        rsync -uvarh --no-g {params.cluster}/results . || echo "No results directory, skipping rsync"
-        rsync -uvarh --no-g {params.cluster}/logs logs || echo "No logs directory, skipping rsync"
+        rsync -uvarh --no-g {params.cluster}/logs . || echo "No logs directory, skipping rsync"
         """

config/config.validation.yaml

@@ -66,7 +66,7 @@ renewable:
     cutout: europe-2019-era5
   hydro:
     cutout: europe-2019-era5
-    flatten_dispatch: 0.2
+    flatten_dispatch: 0.1
 
 conventional:
   unit_commitment: true
@@ -106,5 +106,5 @@ solving:
   options:
     load_shedding: true
     rolling_horizon: false
-    horizon: 389
+    horizon: 1000
-    overlap: 24
+    overlap: 48

data/override_component_attrs/buses.csv

@@ -1,3 +0,0 @@
-attribute,type,unit,default,description,status
-location,string,n/a,n/a,Reference to original electricity bus,Input (optional)
-unit,string,n/a,MWh,Unit of the bus (descriptive only), Input (optional)

data/override_component_attrs/generators.csv

@@ -1,4 +0,0 @@
-attribute,type,unit,default,description,status
-carrier,string,n/a,n/a,carrier,Input (optional)
-lifetime,float,years,inf,lifetime,Input (optional)
-build_year,int,year ,0,build year,Input (optional)

data/override_component_attrs/links.csv

@@ -1,13 +0,0 @@
-attribute,type,unit,default,description,status
-bus2,string,n/a,n/a,2nd bus,Input (optional)
-bus3,string,n/a,n/a,3rd bus,Input (optional)
-bus4,string,n/a,n/a,4th bus,Input (optional)
-efficiency2,static or series,per unit,1,2nd bus efficiency,Input (optional)
-efficiency3,static or series,per unit,1,3rd bus efficiency,Input (optional)
-efficiency4,static or series,per unit,1,4th bus efficiency,Input (optional)
-p2,series,MW,0,2nd bus output,Output
-p3,series,MW,0,3rd bus output,Output
-p4,series,MW,0,4th bus output,Output
-carrier,string,n/a,n/a,carrier,Input (optional)
-lifetime,float,years,inf,lifetime,Input (optional)
-build_year,int,year ,0,build year,Input (optional)

data/override_component_attrs/loads.csv

@@ -1,2 +0,0 @@
-attribute,type,unit,default,description,status
-carrier,string,n/a,n/a,carrier,Input (optional)

data/override_component_attrs/stores.csv

@@ -1,4 +0,0 @@
-attribute,type,unit,default,description,status
-carrier,string,n/a,n/a,carrier,Input (optional)
-lifetime,float,years,inf,lifetime,Input (optional)
-build_year,int,year ,0,build year,Input (optional)

doc/release_notes.rst

@@ -40,11 +40,15 @@ Upcoming Release
   e.g. by setting ``solving: options: transmission_losses: 2`` for an
   approximation with two tangents.
 
+* Handling networks with links with multiple inputs/outputs no longer requires
+  to override component attributes.
+
 * Added configuration option ``enable: retrieve:`` to control whether data
   retrieval rules from snakemake are enabled or not. Th default setting ``auto``
   will automatically detect and enable/disable the rules based on internet connectivity.
 
 
+
 PyPSA-Eur 0.8.0 (18th March 2023)
 =================================
 

envs/environment.yaml

@@ -10,7 +10,6 @@ dependencies:
 - python>=3.8
 - pip
 
-# - pypsa>=0.23
 - atlite>=0.2.9
 - dask
 

rules/build_sector.smk

@@ -724,7 +724,6 @@ rule prepare_sector_network:
         **build_biomass_transport_costs_output,
         **gas_infrastructure,
         **build_sequestration_potentials_output,
-        overrides="data/override_component_attrs",
         network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
         energy_totals_name=RESOURCES + "energy_totals.csv",
         eurostat=input_eurostat,

rules/postprocess.smk

@@ -13,7 +13,6 @@ rule plot_network:
         foresight=config["foresight"],
         plotting=config["plotting"],
     input:
-        overrides="data/override_component_attrs",
         network=RESULTS
         + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
         regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
@@ -76,7 +75,6 @@ rule make_summary:
         scenario=config["scenario"],
         RDIR=RDIR,
     input:
-        overrides="data/override_component_attrs",
         networks=expand(
             RESULTS
             + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",

rules/solve_myopic.smk

@@ -10,7 +10,6 @@ rule add_existing_baseyear:
         existing_capacities=config["existing_capacities"],
         costs=config["costs"],
     input:
-        overrides="data/override_component_attrs",
         network=RESULTS
         + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
         powerplants=RESOURCES + "powerplants.csv",
@@ -52,7 +51,6 @@ rule add_brownfield:
         H2_retrofit_capacity_per_CH4=config["sector"]["H2_retrofit_capacity_per_CH4"],
         threshold_capacity=config["existing_capacities"]["threshold_capacity"],
     input:
-        overrides="data/override_component_attrs",
         network=RESULTS
         + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
         network_p=solved_previous_horizon,  #solved network at previous time step
@@ -91,7 +89,6 @@ rule solve_sector_network_myopic:
             "co2_sequestration_potential", 200
         ),
     input:
-        overrides="data/override_component_attrs",
         network=RESULTS
         + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
         costs="data/costs_{planning_horizons}.csv",

rules/solve_overnight.smk

@@ -12,7 +12,6 @@ rule solve_sector_network:
             "co2_sequestration_potential", 200
         ),
     input:
-        overrides="data/override_component_attrs",
         network=RESULTS
         + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
         costs="data/costs_{}.csv".format(config["costs"]["year"]),

scripts/_helpers.py

@@ -72,92 +72,6 @@ def configure_logging(snakemake, skip_handlers=False):
     logging.basicConfig(**kwargs)
 
 
-def load_network(import_name=None, custom_components=None):
-    """
-    Helper for importing a pypsa.Network with additional custom components.
-
-    Parameters
-    ----------
-    import_name : str
-        As in pypsa.Network(import_name)
-    custom_components : dict
-        Dictionary listing custom components.
-        For using ``snakemake.params['override_components']``
-        in ``config/config.yaml`` define:
-
-        .. code:: yaml
-
-            override_components:
-                ShadowPrice:
-                    component: ["shadow_prices","Shadow price for a global constraint.",np.nan]
-                    attributes:
-                    name: ["string","n/a","n/a","Unique name","Input (required)"]
-                    value: ["float","n/a",0.,"shadow value","Output"]
-
-    Returns
-    -------
-    pypsa.Network
-    """
-    import pypsa
-    from pypsa.descriptors import Dict
-
-    override_components = None
-    override_component_attrs = None
-
-    if custom_components is not None:
-        override_components = pypsa.components.components.copy()
-        override_component_attrs = Dict(
-            {k: v.copy() for k, v in pypsa.components.component_attrs.items()}
-        )
-        for k, v in custom_components.items():
-            override_components.loc[k] = v["component"]
-            override_component_attrs[k] = pd.DataFrame(
-                columns=["type", "unit", "default", "description", "status"]
-            )
-            for attr, val in v["attributes"].items():
-                override_component_attrs[k].loc[attr] = val
-
-    return pypsa.Network(
-        import_name=import_name,
-        override_components=override_components,
-        override_component_attrs=override_component_attrs,
-    )
-
-
-def load_network_for_plots(fn, tech_costs, config, combine_hydro_ps=True):
-    import pypsa
-    from add_electricity import load_costs, update_transmission_costs
-
-    n = pypsa.Network(fn)
-
-    n.loads["carrier"] = n.loads.bus.map(n.buses.carrier) + " load"
-    n.stores["carrier"] = n.stores.bus.map(n.buses.carrier)
-
-    n.links["carrier"] = (
-        n.links.bus0.map(n.buses.carrier) + "-" + n.links.bus1.map(n.buses.carrier)
-    )
-    n.lines["carrier"] = "AC line"
-    n.transformers["carrier"] = "AC transformer"
-
-    n.lines["s_nom"] = n.lines["s_nom_min"]
-    n.links["p_nom"] = n.links["p_nom_min"]
-
-    if combine_hydro_ps:
-        n.storage_units.loc[
-            n.storage_units.carrier.isin({"PHS", "hydro"}), "carrier"
-        ] = "hydro+PHS"
-
-    # if the carrier was not set on the heat storage units
-    # bus_carrier = n.storage_units.bus.map(n.buses.carrier)
-    # n.storage_units.loc[bus_carrier == "heat","carrier"] = "water tanks"
-
-    Nyears = n.snapshot_weightings.objective.sum() / 8760.0
-    costs = load_costs(tech_costs, config["costs"], config["electricity"], Nyears)
-    update_transmission_costs(n, costs)
-
-    return n
-
-
 def update_p_nom_max(n):
     # if extendable carriers (solar/onwind/...) have capacity >= 0,
     # e.g. existing assets from the OPSD project are included to the network,
@@ -367,34 +281,6 @@ def mock_snakemake(rulename, configfiles=[], **wildcards):
     return snakemake
 
 
-def override_component_attrs(directory):
-    """
-    Tell PyPSA that links can have multiple outputs by overriding the
-    component_attrs. This can be done for as many buses as you need with format
-    busi for i = 2,3,4,5,.... See https://pypsa.org/doc/components.html#link-
-    with-multiple-outputs-or-inputs.
-
-    Parameters
-    ----------
-    directory : string
-        Folder where component attributes to override are stored
-        analogous to ``pypsa/component_attrs``, e.g. `links.csv`.
-
-    Returns
-    -------
-    Dictionary of overridden component attributes.
-    """
-    attrs = Dict({k: v.copy() for k, v in component_attrs.items()})
-
-    for component, list_name in components.list_name.items():
-        fn = f"{directory}/{list_name}.csv"
-        if os.path.isfile(fn):
-            overrides = pd.read_csv(fn, index_col=0, na_values="n/a")
-            attrs[component] = overrides.combine_first(attrs[component])
-
-    return attrs
-
-
 def generate_periodic_profiles(dt_index, nodes, weekly_profile, localize=None):
     """
     Give a 24*7 long list of weekly hourly profiles, generate this for each

scripts/add_brownfield.py

@@ -16,7 +16,7 @@ idx = pd.IndexSlice
 
 import numpy as np
 import pypsa
-from _helpers import override_component_attrs, update_config_with_sector_opts
+from _helpers import update_config_with_sector_opts
 from add_existing_baseyear import add_build_year_to_new_assets
 
 
@@ -147,12 +147,11 @@ if __name__ == "__main__":
 
     year = int(snakemake.wildcards.planning_horizons)
 
-    overrides = override_component_attrs(snakemake.input.overrides)
+    n = pypsa.Network(snakemake.input.network)
-    n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
 
     add_build_year_to_new_assets(n, year)
 
-    n_p = pypsa.Network(snakemake.input.network_p, override_component_attrs=overrides)
+    n_p = pypsa.Network(snakemake.input.network_p)
 
     add_brownfield(n, n_p, year)
 

scripts/add_electricity.py

@@ -425,6 +425,13 @@ def attach_conventional_generators(
     add_missing_carriers(n, carriers)
     add_co2_emissions(n, costs, carriers)
 
+    # Replace carrier "natural gas" with the respective technology (OCGT or
+    # CCGT) to align with PyPSA names of "carriers" and avoid filtering "natural
+    # gas" powerplants in ppl.query("carrier in @carriers")
+    ppl.loc[ppl["carrier"] == "natural gas", "carrier"] = ppl.loc[
+        ppl["carrier"] == "natural gas", "technology"
+    ]
+
     ppl = (
         ppl.query("carrier in @carriers")
         .join(costs, on="carrier", rsuffix="_r")

scripts/add_existing_baseyear.py

@@ -21,7 +21,7 @@ import country_converter as coco
 import numpy as np
 import pypsa
 import xarray as xr
-from _helpers import override_component_attrs, update_config_with_sector_opts
+from _helpers import update_config_with_sector_opts
 from add_electricity import sanitize_carriers
 from prepare_sector_network import cluster_heat_buses, define_spatial, prepare_costs
 
@@ -129,10 +129,14 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas
         "Oil": "oil",
         "OCGT": "OCGT",
         "CCGT": "CCGT",
-        "Natural Gas": "gas",
         "Bioenergy": "urban central solid biomass CHP",
     }
 
+    # Replace Fueltype "Natural Gas" with the respective technology (OCGT or CCGT)
+    df_agg.loc[df_agg["Fueltype"] == "Natural Gas", "Fueltype"] = df_agg.loc[
+        df_agg["Fueltype"] == "Natural Gas", "Technology"
+    ]
+
     fueltype_to_drop = [
         "Hydro",
         "Wind",
@@ -601,12 +605,13 @@ if __name__ == "__main__":
 
         snakemake = mock_snakemake(
             "add_existing_baseyear",
+            configfiles="config/test/config.myopic.yaml",
             simpl="",
-            clusters="45",
+            clusters="5",
-            ll="v1.0",
+            ll="v1.5",
             opts="",
-            sector_opts="8760H-T-H-B-I-A-solar+p3-dist1",
+            sector_opts="24H-T-H-B-I-A-solar+p3-dist1",
-            planning_horizons=2020,
+            planning_horizons=2030,
         )
 
     logging.basicConfig(level=snakemake.config["logging"]["level"])
@@ -618,8 +623,8 @@ if __name__ == "__main__":
 
     baseyear = snakemake.params.baseyear
 
-    overrides = override_component_attrs(snakemake.input.overrides)
+    n = pypsa.Network(snakemake.input.network)
-    n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
+
     # define spatial resolution of carriers
     spatial = define_spatial(n.buses[n.buses.carrier == "AC"].index, options)
     add_build_year_to_new_assets(n, baseyear)

scripts/build_powerplants.py

@@ -98,13 +98,15 @@ def add_custom_powerplants(ppl, custom_powerplants, custom_ppl_query=False):
 
 
 def replace_natural_gas_technology(df):
-    mapping = {"Steam Turbine": "OCGT", "Combustion Engine": "OCGT"}
+    mapping = {"Steam Turbine": "CCGT", "Combustion Engine": "OCGT"}
-    tech = df.Technology.replace(mapping).fillna("OCGT")
+    tech = df.Technology.replace(mapping).fillna("CCGT")
-    return df.Technology.where(df.Fueltype != "Natural Gas", tech)
+    return df.Technology.mask(df.Fueltype == "Natural Gas", tech)
 
 
 def replace_natural_gas_fueltype(df):
-    return df.Fueltype.where(df.Fueltype != "Natural Gas", df.Technology)
+    return df.Fueltype.mask(
+        (df.Technology == "OCGT") | (df.Technology == "CCGT"), "Natural Gas"
+    )
 
 
 if __name__ == "__main__":

scripts/make_summary.py

@@ -16,7 +16,6 @@ import sys
 import numpy as np
 import pandas as pd
 import pypsa
-from _helpers import override_component_attrs
 from prepare_sector_network import prepare_costs
 
 idx = pd.IndexSlice
@@ -300,9 +299,9 @@ def calculate_energy(n, label, energy):
                 )
                 # remove values where bus is missing (bug in nomopyomo)
                 no_bus = c.df.index[c.df["bus" + port] == ""]
-                totals.loc[no_bus] = n.component_attrs[c.name].loc[
+                totals.loc[no_bus] = float(
-                    "p" + port, "default"
+                    n.component_attrs[c.name].loc["p" + port, "default"]
-                ]
+                )
                 c_energies -= totals.groupby(c.df.carrier).sum()
 
         c_energies = pd.concat([c_energies], keys=[c.list_name])
@@ -659,8 +658,7 @@ def make_summaries(networks_dict):
     for label, filename in networks_dict.items():
         logger.info(f"Make summary for scenario {label}, using {filename}")
 
-        overrides = override_component_attrs(snakemake.input.overrides)
+        n = pypsa.Network(filename)
-        n = pypsa.Network(filename, override_component_attrs=overrides)
 
         assign_carriers(n)
         assign_locations(n)

scripts/plot_network.py

@@ -20,7 +20,6 @@ import geopandas as gpd
 import matplotlib.pyplot as plt
 import pandas as pd
 import pypsa
-from _helpers import override_component_attrs
 from make_summary import assign_carriers
 from plot_summary import preferred_order, rename_techs
 from pypsa.plot import add_legend_circles, add_legend_lines, add_legend_patches
@@ -930,8 +929,7 @@ if __name__ == "__main__":
 
     logging.basicConfig(level=snakemake.config["logging"]["level"])
 
-    overrides = override_component_attrs(snakemake.input.overrides)
+    n = pypsa.Network(snakemake.input.network)
-    n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
 
     regions = gpd.read_file(snakemake.input.regions).set_index("name")
 

scripts/prepare_sector_network.py

@@ -17,11 +17,7 @@ import numpy as np
 import pandas as pd
 import pypsa
 import xarray as xr
-from _helpers import (
+from _helpers import generate_periodic_profiles, update_config_with_sector_opts
-    generate_periodic_profiles,
-    override_component_attrs,
-    update_config_with_sector_opts,
-)
 from add_electricity import calculate_annuity, sanitize_carriers
 from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2
 from networkx.algorithms import complement
@@ -3279,8 +3275,7 @@ if __name__ == "__main__":
 
     investment_year = int(snakemake.wildcards.planning_horizons[-4:])
 
-    overrides = override_component_attrs(snakemake.input.overrides)
+    n = pypsa.Network(snakemake.input.network)
-    n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
 
     pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0)
     nhours = n.snapshot_weightings.generators.sum()

scripts/solve_network.py

@@ -33,11 +33,7 @@ import numpy as np
 import pandas as pd
 import pypsa
 import xarray as xr
-from _helpers import (
+from _helpers import configure_logging, update_config_with_sector_opts
-    configure_logging,
-    override_component_attrs,
-    update_config_with_sector_opts,
-)
 
 logger = logging.getLogger(__name__)
 pypsa.pf.logger.setLevel(logging.WARNING)
@@ -672,11 +668,7 @@ if __name__ == "__main__":
 
     np.random.seed(solve_opts.get("seed", 123))
 
-    if "overrides" in snakemake.input.keys():
+    n = pypsa.Network(snakemake.input.network)
-        overrides = override_component_attrs(snakemake.input.overrides)
-        n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
-    else:
-        n = pypsa.Network(snakemake.input.network)
 
     n = prepare_network(
         n,

scripts/solve_operations_network.py

@@ -11,11 +11,7 @@ import logging
 
 import numpy as np
 import pypsa
-from _helpers import (
+from _helpers import configure_logging, update_config_with_sector_opts
-    configure_logging,
-    override_component_attrs,
-    update_config_with_sector_opts,
-)
 from solve_network import prepare_network, solve_network
 
 logger = logging.getLogger(__name__)
@@ -45,11 +41,7 @@ if __name__ == "__main__":
 
     np.random.seed(solve_opts.get("seed", 123))
 
-    if "overrides" in snakemake.input:
+    n = pypsa.Network(snakemake.input.network)
-        overrides = override_component_attrs(snakemake.input.overrides)
-        n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides)
-    else:
-        n = pypsa.Network(snakemake.input.network)
 
     n.optimize.fix_optimal_capacities()
     n = prepare_network(n, solve_opts, config=snakemake.config)