draf validate rule & plot statistics rule

rules/collect.smk

@@ -73,3 +73,12 @@ rule plot_networks:
             + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf",
             **config["scenario"]
         ),
+
+
+rule validate_elec_networks:
+    input:
+        expand(
+            RESULTS
+            + "figures/validate_electricity_production_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
+            **config["scenario"]
+        ),

rules/postprocess.smk

@@ -146,3 +146,14 @@ rule plot_summary:
         "../envs/environment.yaml"
     script:
         "../scripts/plot_summary.py"
+
+
+rule plot_statistics:
+    input:
+        overrides="data/override_component_attrs",
+        network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
+    output:
+        bar=RESULTS
+        + "figures/statistics_bar_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.pdf",
+    script:
+        "../scripts/plot_statistics.py"

rules/validate.smk

@@ -0,0 +1,32 @@
+# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+
+
+rule build_electricity_production:
+    """
+    This rule builds the electricity production for each country and technology from ENTSO-E data.
+    The data is used for validation of the optimization results.
+    """
+    params:
+        snapshots=config["snapshots"],
+        countries=config["countries"],
+    output:
+        RESOURCES + "historical_electricity_production.csv",
+    log:
+        LOGS + "build_electricity_production.log",
+    resources:
+        mem_mb=5000,
+    script:
+        "../scripts/retrieve_electricity_production.py"
+
+
+rule plot_electricity_production:
+    input:
+        network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
+        electricity_production="data/historical_electricity_production.csv",
+    output:
+        electricity_producion=RESULTS
+        + "figures/validate_electricity_production_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.pdf",
+    script:
+        "scripts/plot_electricity_production.py"

scripts/build_electricity_production.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+"""
+Created on Mon Jul  3 11:19:54 2023.
+
+@author: fabian
+"""
+
+
+import logging
+
+import pandas as pd
+from entsoe import EntsoePandasClient
+from entsoe.exceptions import NoMatchingDataError
+
+logger = logging.getLogger(__name__)
+
+
+carrier_grouper = {
+    "Waste": "Biomass",
+    "Hydro Pumped Storage": "Hydro",
+    "Hydro Water Reservoir": "Hydro",
+    "Hydro Run-of-river and poundage": "Run of River",
+    "Fossil Coal-derived gas": "Gas",
+    "Fossil Gas": "Gas",
+    "Fossil Oil": "Oil",
+    "Fossil Oil shale": "Oil",
+    "Fossil Brown coal/Lignite": "Lignite",
+    "Fossil Peat": "Lignite",
+    "Fossil Hard coal": "Coal",
+}
+
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake("retrieve_historical_electricity_generation")
+
+
+api_key = "aeff3346-a240-40df-bd12-692772b845d0"
+client = EntsoePandasClient(api_key=api_key)
+
+start = pd.Timestamp(snakemake.params.snapshots["start"], tz="Europe/Brussels")
+end = pd.Timestamp(snakemake.params.snapshots["end"], tz="Europe/Brussels")
+
+countries = snakemake.params.countries
+
+
+generation = []
+unavailable_countries = []
+
+for country in countries:
+    country_code = country
+
+    try:
+        gen = client.query_generation(country, start=start, end=end, nett=True)
+        gen = gen.tz_localize(None).resample("1h").mean()
+        gen = gen.rename(columns=carrier_grouper).groupby(level=0, axis=1).sum()
+        generation.append(gen)
+    except NoMatchingDataError:
+        unavailable_countries.append(country)
+
+
+if unavailable_countries:
+    logger.warning(
+        f"Historical electricity production for countries {', '.join(unavailable_countries)} not available."
+    )
+
+keys = [c for c in countries if c not in unavailable_countries]
+generation = pd.concat(generation, keys=keys, axis=1)
+generation = generation.loc[start.tz_localize(None) : end.tz_localize(None)]
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+generation.to_csv(snakemake.output[0])

scripts/plot_historic_comparison.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+"""
+Created on Mon Jul  3 12:50:26 2023.
+
+@author: fabian
+"""
+
+import pandas as pd
+import pypsa
+from pypsa.statistics import get_bus_and_carrier
+
+carrier_groups = {
+    "Offshore Wind (AC)": "Offshore Wind",
+    "Offshore Wind (DC)": "Offshore Wind",
+    "Open-Cycle Gas": "Gas",
+    "Combined-Cycle Gas": "Gas",
+    "Reservoir & Dam": "Hydro",
+    "Pumped Hydro Storage": "Hydro",
+}
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake(
+            "plot_statistics",
+            simpl="",
+            opts="Co2L-3h",
+            clusters="37c",
+            ll="v1.0",
+        )
+
+n = pypsa.Network(snakemake.input.network)
+historic = pd.read_csv(
+    snakemake.input.historic_electricity_generation, index_col=0, header=[0, 1]
+)
+
+
+simulated = n.statistics.dispatch(groupby=get_bus_and_carrier, aggregate_time=False).T
+simulated = simulated[["Generator", "StorageUnit"]].droplevel(0, axis=1)
+simulated = simulated.rename(columns=n.buses.country, level=0)
+simulated = simulated.rename(carrier_groups, level=1)
+simulated = simulated.groupby(axis=1, level=[0, 1]).sum()

scripts/plot_statistics.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+"""
+Created on Fri Jun 30 10:50:53 2023.
+
+@author: fabian
+"""
+
+import matplotlib.pyplot as plt
+import pypsa
+import seaborn as sns
+
+sns.set_theme("paper", style="whitegrid")
+
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake(
+            "plot_statistics",
+            simpl="",
+            opts="Co2L-3h",
+            clusters="37c",
+            ll="v1.0",
+        )
+
+
+n = pypsa.Network(snakemake.network)
+
+
+n.loads.carrier = "load"
+n.carriers.loc["load", ["nice_name", "color"]] = "Load", "darkred"
+colors = n.carriers.set_index("nice_name").color.where(lambda s: s != "", "lightgrey")
+
+
+def rename_index(ds):
+    return ds.set_axis(ds.index.map(lambda x: f"{x[1]}\n({x[0].lower()})"))
+
+
+def plot_static_per_carrier(ds, ax, drop_zero=True):
+    if drop_zero:
+        ds = ds[ds != 0]
+    ds = ds.dropna()
+    c = colors[ds.index.get_level_values("carrier")]
+    ds = ds.pipe(rename_index)
+    label = f"{ds.attrs['name']} [{ds.attrs['unit']}]"
+    ds.plot.barh(color=c.values, xlabel=label, ax=ax)
+
+
+fig, ax = plt.subplots()
+ds = n.statistics.capacity_factor().dropna()
+plot_static_per_carrier(ds, ax)
+# fig.savefig("")
+
+fig, ax = plt.subplots()
+ds = n.statistics.installed_capacity().dropna()
+ds = ds.drop("Line")
+ds = ds / 1e3
+ds.attrs["unit"] = "GW"
+plot_static_per_carrier(ds, ax)
+# fig.savefig("")
+
+
+fig, ax = plt.subplots()
+ds = n.statistics.optimal_capacity()
+ds = ds.drop("Line")
+ds = ds / 1e3
+ds.attrs["unit"] = "GW"
+plot_static_per_carrier(ds, ax)
+# fig.savefig("")
+
+
+fig, ax = plt.subplots()
+ds = n.statistics.capex()
+plot_static_per_carrier(ds, ax)
+# fig.savefig("")
+
+fig, ax = plt.subplots()
+ds = n.statistics.curtailment()
+plot_static_per_carrier(ds, ax)
+# fig.savefig("")
+
+fig, ax = plt.subplots()
+ds = n.statistics.supply()
+ds = ds.drop("Line")
+ds = ds / 1e6
+ds.attrs["unit"] = "TWh"
+plot_static_per_carrier(ds, ax)
+# fig.savefig("")
+
+
+fig, ax = plt.subplots()
+ds = n.statistics.withdrawal()
+ds = ds.drop("Line")
+ds = ds / -1e6
+ds.attrs["unit"] = "TWh"
+plot_static_per_carrier(ds, ax)
+# fig.savefig("")